From 5fb66f864a8a6c6f2257df071d9b58ea06226a8a Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Thu, 14 Apr 2022 10:32:48 +0300 Subject: [PATCH 01/74] add `--sync` program option (#1164) --- core/application/app_configuration.hpp | 6 ++++ .../impl/app_configuration_impl.cpp | 33 +++++++++++++++++++ .../impl/app_configuration_impl.hpp | 4 +++ .../application/app_configuration_mock.hpp | 2 ++ 4 files changed, 45 insertions(+) diff --git a/core/application/app_configuration.hpp b/core/application/app_configuration.hpp index b075c24276..2f1af6f4f8 100644 --- a/core/application/app_configuration.hpp +++ b/core/application/app_configuration.hpp @@ -194,6 +194,12 @@ namespace kagome::application { virtual const std::vector &telemetryEndpoints() const = 0; + enum class SyncMethod { Full, Fast }; + /** + * @return enum constant of the chosen sync method + */ + virtual SyncMethod syncMethod() const = 0; + enum class RuntimeExecutionMethod { Compile, Interpret }; /** * @return enum constant of the chosen runtime backend diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index c8b6892a49..7f9499e35a 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -55,6 +55,8 @@ namespace { roles.flags.full = 1; return roles; }(); + const auto def_sync_method = + kagome::application::AppConfiguration::SyncMethod::Full; const auto def_runtime_exec_method = kagome::application::AppConfiguration::RuntimeExecutionMethod::Interpret; const auto def_use_wavm_cache_ = false; @@ -70,6 +72,7 @@ namespace { const uint32_t def_in_peers = 25; const uint32_t def_in_peers_light = 100; const uint32_t def_random_walk_interval = 15; + const auto def_full_sync = "Full"; const auto def_wasm_execution = "Interpreted"; /** @@ -89,6 +92,18 @@ namespace { return name; } + std::optional + str_to_sync_method(std::string_view str) { + using SM = kagome::application::AppConfiguration::SyncMethod; + if (str == "Full") { + return SM::Full; + } + if (str == "Fast") { + return SM::Fast; + } + return std::nullopt; + } + std::optional str_to_runtime_exec_method(std::string_view str) { using REM = kagome::application::AppConfiguration::RuntimeExecutionMethod; @@ -155,6 +170,7 @@ namespace kagome::application { node_version_(buildVersion()), max_ws_connections_(def_ws_max_connections), random_walk_interval_(def_random_walk_interval), + sync_method_{def_sync_method}, runtime_exec_method_{def_runtime_exec_method}, use_wavm_cache_(def_use_wavm_cache_), purge_wavm_cache_(def_purge_wavm_cache_), @@ -697,6 +713,8 @@ namespace kagome::application { development_desc.add_options() ("dev", "if node run in development mode") ("dev-with-wipe", "if needed to wipe base path (only for dev mode)") + ("sync", po::value()->default_value(def_full_sync), + "choose the desired sync method (Full, Fast). Full is used by default.") ("wasm-execution", po::value()->default_value(def_wasm_execution), "choose the desired wasm execution method (Compiled, Interpreted)") ("unsafe-cached-wavm-runtime", "use WAVM runtime cache") @@ -1080,6 +1098,21 @@ namespace kagome::application { } } + bool sync_method_value_error = false; + find_argument( + vm, "sync", [this, &sync_method_value_error](std::string const &val) { + auto sync_method_opt = str_to_sync_method(val); + if (not sync_method_opt) { + sync_method_value_error = true; + SL_ERROR(logger_, "Invalid sync method specified: '{}'", val); + } else { + sync_method_ = sync_method_opt.value(); + } + }); + if (sync_method_value_error) { + return false; + } + bool exec_method_value_error = false; find_argument( vm, diff --git a/core/application/impl/app_configuration_impl.hpp b/core/application/impl/app_configuration_impl.hpp index 1663034ce8..43ef8b1c81 100644 --- a/core/application/impl/app_configuration_impl.hpp +++ b/core/application/impl/app_configuration_impl.hpp @@ -151,6 +151,9 @@ namespace kagome::application { const override { return telemetry_endpoints_; } + SyncMethod syncMethod() const override { + return sync_method_; + } RuntimeExecutionMethod runtimeExecMethod() const override { return runtime_exec_method_; } @@ -297,6 +300,7 @@ namespace kagome::application { std::string node_version_; uint32_t max_ws_connections_; uint32_t random_walk_interval_; + SyncMethod sync_method_; RuntimeExecutionMethod runtime_exec_method_; bool use_wavm_cache_; bool purge_wavm_cache_; diff --git a/test/mock/core/application/app_configuration_mock.hpp b/test/mock/core/application/app_configuration_mock.hpp index b1ce67a6ec..4791a89747 100644 --- a/test/mock/core/application/app_configuration_mock.hpp +++ b/test/mock/core/application/app_configuration_mock.hpp @@ -112,6 +112,8 @@ namespace kagome::application { (), (const, override)); + MOCK_METHOD(AppConfiguration::SyncMethod, syncMethod, (), (const, override)); + MOCK_METHOD(AppConfiguration::RuntimeExecutionMethod, runtimeExecMethod, (), From 6b67c6044b3e1648fda0b868ee8f6d0ed779b106 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Fri, 15 Apr 2022 09:17:52 +0300 Subject: [PATCH 02/74] implement block appender for fast sync (#1164) --- core/consensus/babe/block_appender.hpp | 23 ++ core/consensus/babe/impl/CMakeLists.txt | 13 ++ .../babe/impl/block_appender_impl.cpp | 220 ++++++++++++++++++ .../babe/impl/block_appender_impl.hpp | 63 +++++ core/injector/CMakeLists.txt | 1 + core/injector/application_injector.cpp | 2 + core/log/configurator.cpp | 1 + core/network/impl/synchronizer_impl.cpp | 34 ++- core/network/impl/synchronizer_impl.hpp | 9 + test/core/network/synchronizer_test.cpp | 9 +- .../consensus/babe/block_appender_mock.hpp | 28 +++ 11 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 core/consensus/babe/block_appender.hpp create mode 100644 core/consensus/babe/impl/block_appender_impl.cpp create mode 100644 core/consensus/babe/impl/block_appender_impl.hpp create mode 100644 test/mock/core/consensus/babe/block_appender_mock.hpp diff --git a/core/consensus/babe/block_appender.hpp b/core/consensus/babe/block_appender.hpp new file mode 100644 index 0000000000..9a4b57c11f --- /dev/null +++ b/core/consensus/babe/block_appender.hpp @@ -0,0 +1,23 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CONSENSUS_BLOCKAPPENDER +#define KAGOME_CONSENSUS_BLOCKAPPENDER + +#include "outcome/outcome.hpp" +#include "primitives/block_data.hpp" + +namespace kagome::consensus { + + class BlockAppender { + public: + virtual ~BlockAppender() = default; + + virtual outcome::result appendBlock(primitives::BlockData &&b) = 0; + }; + +} // namespace kagome::consensus + +#endif // KAGOME_CONSENSUS_BLOCKAPPENDER diff --git a/core/consensus/babe/impl/CMakeLists.txt b/core/consensus/babe/impl/CMakeLists.txt index 4acc5d406d..0bb62676db 100644 --- a/core/consensus/babe/impl/CMakeLists.txt +++ b/core/consensus/babe/impl/CMakeLists.txt @@ -30,6 +30,19 @@ target_link_libraries(babe_digests_util block_storage_error ) +add_library(block_appender + block_appender_impl.cpp + ) +target_link_libraries(block_appender + logger + primitives + scale::scale + block_tree_error + threshold_util + babe_digests_util + blockchain_common + ) + add_library(block_executor block_executor_impl.cpp ) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp new file mode 100644 index 0000000000..74e93fa779 --- /dev/null +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -0,0 +1,220 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consensus/babe/impl/block_appender_impl.hpp" + +#include + +#include "blockchain/block_tree_error.hpp" +#include "blockchain/impl/common.hpp" +#include "consensus/babe/impl/babe_digests_util.hpp" +#include "consensus/babe/impl/threshold_util.hpp" +#include "consensus/babe/types/slot.hpp" +#include "network/helpers/peer_id_formatter.hpp" +#include "primitives/common.hpp" +#include "scale/scale.hpp" +#include "transaction_pool/transaction_pool_error.hpp" + +OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus, BlockAppenderImpl::Error, e) { + using E = kagome::consensus::BlockAppenderImpl::Error; + switch (e) { + case E::INVALID_BLOCK: + return "Invalid block"; + case E::PARENT_NOT_FOUND: + return "Parent not found"; + } + return "Unknown error"; +} + +namespace { + constexpr const char *kBlockExecutionTime = + "kagome_block_verification_and_import_time"; +} + +namespace kagome::consensus { + + BlockAppenderImpl::BlockAppenderImpl( + std::shared_ptr block_tree, + std::shared_ptr configuration, + std::shared_ptr block_validator, + std::shared_ptr grandpa_environment, + std::shared_ptr hasher, + std::shared_ptr + authority_update_observer, + std::shared_ptr babe_util) + : block_tree_{std::move(block_tree)}, + babe_configuration_{std::move(configuration)}, + block_validator_{std::move(block_validator)}, + grandpa_environment_{std::move(grandpa_environment)}, + hasher_{std::move(hasher)}, + authority_update_observer_{std::move(authority_update_observer)}, + babe_util_(std::move(babe_util)), + logger_{log::createLogger("BlockAppender", "block_appender")} { + BOOST_ASSERT(block_tree_ != nullptr); + BOOST_ASSERT(babe_configuration_ != nullptr); + BOOST_ASSERT(block_validator_ != nullptr); + BOOST_ASSERT(grandpa_environment_ != nullptr); + BOOST_ASSERT(hasher_ != nullptr); + BOOST_ASSERT(authority_update_observer_ != nullptr); + BOOST_ASSERT(babe_util_ != nullptr); + BOOST_ASSERT(logger_ != nullptr); + } + + outcome::result BlockAppenderImpl::appendBlock( + primitives::BlockData &&b) { + if (not b.header.has_value()) { + logger_->warn("Skipping a block without header"); + return Error::INVALID_BLOCK; + } + auto &header = b.header.value(); + + if (auto header_res = block_tree_->getBlockHeader(header.parent_hash); + header_res.has_error() + && header_res.error() == blockchain::BlockTreeError::HEADER_NOT_FOUND) { + logger_->warn("Skipping a block with unknown parent"); + return Error::PARENT_NOT_FOUND; + } else if (header_res.has_error()) { + return header_res.as_failure(); + } + + // get current time to measure performance if block execution + auto t_start = std::chrono::high_resolution_clock::now(); + + auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); + + bool block_already_exists = false; + + // check if block header already exists. If so, do not append + if (auto header_res = block_tree_->getBlockHeader(block_hash); + header_res.has_value()) { + SL_DEBUG(logger_, + "Skip existing block: {}", + primitives::BlockInfo(header.number, block_hash)); + + OUTCOME_TRY(block_tree_->addExistingBlock(block_hash, header)); + block_already_exists = true; + } else if (header_res.error() != blockchain::BlockTreeError::HEADER_NOT_FOUND) { + return header_res.as_failure(); + } + + primitives::Block block{.header = std::move(header)}; + + OUTCOME_TRY(babe_digests, getBabeDigests(block.header)); + + const auto &babe_header = babe_digests.second; + + auto slot_number = babe_header.slot_number; + auto epoch_number = babe_util_->slotToEpoch(slot_number); + + logger_->info( + "Applying block {} ({} in slot {}, epoch {})", // + primitives::BlockInfo(block.header.number, block_hash), + babe_header.slotType() == SlotType::Primary ? "primary" + : babe_header.slotType() == SlotType::SecondaryVRF ? "secondary-vrf" + : babe_header.slotType() == SlotType::SecondaryPlain ? "secondary-plain" + : "unknown", + slot_number, + epoch_number); + + OUTCOME_TRY( + this_block_epoch_descriptor, + block_tree_->getEpochDigest(epoch_number, block.header.parent_hash)); + + SL_TRACE(logger_, + "Actual epoch digest to apply block {} (slot {}, epoch {}). " + "Randomness: {}", + primitives::BlockInfo(block.header.number, block_hash), + slot_number, + epoch_number, + this_block_epoch_descriptor.randomness); + + auto threshold = calculateThreshold(babe_configuration_->leadership_rate, + this_block_epoch_descriptor.authorities, + babe_header.authority_index); + + OUTCOME_TRY(block_validator_->validateHeader( + block.header, + epoch_number, + this_block_epoch_descriptor.authorities[babe_header.authority_index].id, + threshold, + this_block_epoch_descriptor.randomness)); + + if (auto next_epoch_digest_res = getNextEpochDigest(block.header)) { + auto &next_epoch_digest = next_epoch_digest_res.value(); + SL_VERBOSE(logger_, + "Next epoch digest has gotten in block {} (slot {}). " + "Randomness: {}", + slot_number, + primitives::BlockInfo(block.header.number, block_hash), + next_epoch_digest.randomness); + } + + auto block_without_seal_digest = block; + + // block should be applied without last digest which contains the seal + block_without_seal_digest.header.digest.pop_back(); + + auto parent = block_tree_->getBlockHeader(block.header.parent_hash).value(); + + if (not block_already_exists) { + OUTCOME_TRY(block_tree_->addBlockHeader(block.header)); + } + + // observe possible changes of authorities + // (must be done strictly after block will be added) + for (auto &digest_item : block_without_seal_digest.header.digest) { + auto res = visit_in_place( + digest_item, + [&](const primitives::Consensus &consensus_message) + -> outcome::result { + return authority_update_observer_->onConsensus( + primitives::BlockInfo{block.header.number, block_hash}, + consensus_message); + }, + [](const auto &) { return outcome::success(); }); + if (res.has_error()) { + rollbackBlock(block_hash); + return res.as_failure(); + } + } + + // apply justification if any (must be done strictly after block will be + // added and his consensus-digests will be handled) + if (b.justification.has_value()) { + SL_VERBOSE(logger_, + "Justification received for block {}", + primitives::BlockInfo(block.header.number, block_hash)); + auto res = grandpa_environment_->applyJustification( + primitives::BlockInfo(block.header.number, block_hash), + b.justification.value()); + if (res.has_error()) { + rollbackBlock(block_hash); + return res.as_failure(); + } + } + + auto t_end = std::chrono::high_resolution_clock::now(); + + logger_->info( + "Imported block {} within {} ms", + primitives::BlockInfo(block.header.number, block_hash), + std::chrono::duration_cast(t_end - t_start) + .count()); + + return outcome::success(); + } + + void BlockAppenderImpl::rollbackBlock( + const primitives::BlockHash &block_hash) { + auto removal_res = block_tree_->removeLeaf(block_hash); + if (removal_res.has_error()) { + SL_WARN(logger_, + "Rolling back of block {} is failed: {}", + block_hash, + removal_res.error().message()); + } + } + + } // namespace kagome::consensus diff --git a/core/consensus/babe/impl/block_appender_impl.hpp b/core/consensus/babe/impl/block_appender_impl.hpp new file mode 100644 index 0000000000..f5bc78fce2 --- /dev/null +++ b/core/consensus/babe/impl/block_appender_impl.hpp @@ -0,0 +1,63 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CONSENSUS_BLOCKAPPENDERIMPL +#define KAGOME_CONSENSUS_BLOCKAPPENDERIMPL + +#include "consensus/babe/block_appender.hpp" + +#include + +#include "blockchain/block_tree.hpp" +#include "clock/timer.hpp" +#include "consensus/authority/authority_update_observer.hpp" +#include "consensus/babe/babe_util.hpp" +#include "consensus/grandpa/environment.hpp" +#include "consensus/validation/block_validator.hpp" +#include "crypto/hasher.hpp" +#include "log/logger.hpp" +#include "primitives/babe_configuration.hpp" +#include "primitives/block_header.hpp" + +namespace kagome::consensus { + + class BlockAppenderImpl + : public BlockAppender, + public std::enable_shared_from_this { + public: + enum class Error { INVALID_BLOCK = 1, PARENT_NOT_FOUND }; + + BlockAppenderImpl( + std::shared_ptr block_tree, + std::shared_ptr configuration, + std::shared_ptr block_validator, + std::shared_ptr grandpa_environment, + std::shared_ptr hasher, + std::shared_ptr + authority_update_observer, + std::shared_ptr babe_util); + + outcome::result appendBlock(primitives::BlockData &&b) override; + + private: + void rollbackBlock(const primitives::BlockHash &block_hash); + + std::shared_ptr block_tree_; + std::shared_ptr babe_configuration_; + std::shared_ptr block_validator_; + std::shared_ptr grandpa_environment_; + std::shared_ptr hasher_; + std::shared_ptr + authority_update_observer_; + std::shared_ptr babe_util_; + + log::Logger logger_; + }; + +} // namespace kagome::consensus + +OUTCOME_HPP_DECLARE_ERROR(kagome::consensus, BlockAppenderImpl::Error); + +#endif // KAGOME_CONSENSUS_BLOCKAPPENDERIMPL diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index 07c9566042..3290ac8f63 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -32,6 +32,7 @@ target_link_libraries(application_injector binaryen_module_factory bip39_provider block_announce_protocol + block_appender block_builder_api block_executor block_header_repository diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 4832097622..c7386be1a0 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -60,6 +60,7 @@ #include "consensus/babe/impl/babe_impl.hpp" #include "consensus/babe/impl/babe_lottery_impl.hpp" #include "consensus/babe/impl/babe_util_impl.hpp" +#include "consensus/babe/impl/block_appender_impl.hpp" #include "consensus/babe/impl/block_executor_impl.hpp" #include "consensus/babe/impl/consistency_keeper_impl.hpp" #include "consensus/grandpa/impl/environment_impl.hpp" @@ -1265,6 +1266,7 @@ namespace { di::bind.to( [](auto const &injector) { return get_peer_manager(injector); }), di::bind.template to(), + di::bind.template to(), di::bind.to( [](auto const &injector) { return get_block_executor(injector); }), di::bind.to( diff --git a/core/log/configurator.cpp b/core/log/configurator.cpp index 8c99250bca..f1fa8a52cf 100644 --- a/core/log/configurator.cpp +++ b/core/log/configurator.cpp @@ -50,6 +50,7 @@ namespace kagome::log { - name: babe children: - name: babe_lottery + - name: block_appender - name: block_executor - name: block_validator - name: grandpa diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 3fe0ad7b5b..05c40d6097 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -7,9 +7,9 @@ #include +#include "application/app_configuration.hpp" #include "blockchain/block_tree_error.hpp" #include "network/helpers/peer_id_formatter.hpp" -#include "network/types/block_attributes.hpp" #include "primitives/common.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerImpl::Error, e) { @@ -44,18 +44,35 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerImpl::Error, e) { namespace { constexpr const char *kImportQueueLength = "kagome_import_queue_blocks_submitted"; -} + + kagome::network::BlockAttributes attributesForSync( + kagome::application::AppConfiguration::SyncMethod method) { + using SM = kagome::application::AppConfiguration::SyncMethod; + switch (method) { + case SM::Full: + return kagome::network::BlocksRequest::kBasicAttributes; + case SM::Fast: + return kagome::network::BlockAttribute::HEADER + | kagome::network::BlockAttribute::JUSTIFICATION; + } + return kagome::network::BlocksRequest::kBasicAttributes; + } +} // namespace namespace kagome::network { SynchronizerImpl::SynchronizerImpl( + const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr block_tree, + std::shared_ptr block_appender, std::shared_ptr block_executor, std::shared_ptr router, std::shared_ptr scheduler, std::shared_ptr hasher) - : block_tree_(std::move(block_tree)), + : app_config_(app_config), + block_tree_(std::move(block_tree)), + block_appender_(std::move(block_appender)), block_executor_(std::move(block_executor)), router_(std::move(router)), scheduler_(std::move(scheduler)), @@ -514,9 +531,7 @@ namespace kagome::network { return; } - network::BlocksRequest request{network::BlockAttribute::HEADER - | network::BlockAttribute::BODY - | network::BlockAttribute::JUSTIFICATION, + network::BlocksRequest request{attributesForSync(app_config_.syncMethod()), from.hash, std::nullopt, network::Direction::ASCENDING, @@ -906,7 +921,12 @@ namespace kagome::network { } else { const BlockInfo block_info(block.header->number, block.hash); - auto applying_res = block_executor_->applyBlock(std::move(block)); + auto applying_res = + app_config_.syncMethod() + == application::AppConfiguration::SyncMethod::Full + ? block_executor_->applyBlock(std::move(block)) + : block_appender_->appendBlock(std::move(block)); + notifySubscribers({number, hash}, applying_res); if (not applying_res.has_value()) { diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 01c60d1577..04fb28b17a 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -15,10 +15,15 @@ #include "application/app_state_manager.hpp" #include "consensus/babe/block_executor.hpp" +#include "consensus/babe/block_appender.hpp" #include "metrics/metrics.hpp" #include "network/router.hpp" #include "telemetry/service.hpp" +namespace kagome::application { + class AppConfiguration; +} + namespace kagome::network { class SynchronizerImpl @@ -54,8 +59,10 @@ namespace kagome::network { }; SynchronizerImpl( + const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr block_tree, + std::shared_ptr block_appender, std::shared_ptr block_executor, std::shared_ptr router, std::shared_ptr scheduler, @@ -149,7 +156,9 @@ namespace kagome::network { const libp2p::peer::PeerId &peer_id, const BlocksRequest::Fingerprint &fingerprint); + const application::AppConfiguration &app_config_; std::shared_ptr block_tree_; + std::shared_ptr block_appender_; std::shared_ptr block_executor_; std::shared_ptr router_; std::shared_ptr scheduler_; diff --git a/test/core/network/synchronizer_test.cpp b/test/core/network/synchronizer_test.cpp index 9a145fa5ab..9ee8f68972 100644 --- a/test/core/network/synchronizer_test.cpp +++ b/test/core/network/synchronizer_test.cpp @@ -9,8 +9,10 @@ #include #include +#include "mock/core/application/app_configuration_mock.hpp" #include "mock/core/application/app_state_manager_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" +#include "mock/core/consensus/babe/block_appender_mock.hpp" #include "mock/core/consensus/babe/block_executor_mock.hpp" #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/network/protocols/sync_protocol_mock.hpp" @@ -65,18 +67,23 @@ class SynchronizerTest EXPECT_CALL(*scheduler, scheduleImplMockCall(_, _, _)).Times(AnyNumber()); synchronizer = - std::make_shared(app_state_manager, + std::make_shared(app_config, + app_state_manager, block_tree, + block_appender, block_executor, router, scheduler, hasher); } + application::AppConfigurationMock app_config; std::shared_ptr app_state_manager = std::make_shared(); std::shared_ptr block_tree = std::make_shared(); + std::shared_ptr block_appender = + std::make_shared(); std::shared_ptr block_executor = std::make_shared(); std::shared_ptr sync_protocol = diff --git a/test/mock/core/consensus/babe/block_appender_mock.hpp b/test/mock/core/consensus/babe/block_appender_mock.hpp new file mode 100644 index 0000000000..a519ef45b3 --- /dev/null +++ b/test/mock/core/consensus/babe/block_appender_mock.hpp @@ -0,0 +1,28 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CONSENSUS_BLOCKAPPENDERMOCK +#define KAGOME_CONSENSUS_BLOCKAPPENDERMOCK + +#include "consensus/babe/block_appender.hpp" + +#include + +namespace kagome::consensus { + + class BlockAppenderMock : public BlockAppender { + public: + MOCK_METHOD(outcome::result, + appendBlock, + (const primitives::BlockData &block), + ()); + outcome::result appendBlock(primitives::BlockData &&block) override { + return appendBlock(block); + } + }; + +} // namespace kagome::consensus + +#endif // KAGOME_CONSENSUS_BLOCKAPPENDERMOCK From 82464b9c115ea677c1782378e4c53732de0a4e0e Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Tue, 19 Apr 2022 10:28:47 +0300 Subject: [PATCH 03/74] apply sync state and start validating (#1164) apply sync state and start validating (#1164) move syncState call into babe (#1164) --- core/consensus/babe/impl/babe_impl.cpp | 20 +++-- .../babe/impl/block_executor_impl.cpp | 10 +-- .../adapters/protobuf_state_response.hpp | 16 ++++ core/network/impl/synchronizer_impl.cpp | 80 +++++++++++++++++-- core/network/impl/synchronizer_impl.hpp | 19 ++++- core/network/synchronizer.hpp | 14 ++-- .../consensus/babe/block_executor_test.cpp | 4 +- test/core/network/synchronizer_test.cpp | 8 ++ .../types/protobuf_state_request_test.cpp | 4 +- test/mock/core/host_api/host_api_mock.hpp | 1 + test/mock/core/network/synchronizer_mock.hpp | 15 ++++ 11 files changed, 164 insertions(+), 27 deletions(-) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 7a9e35d02e..79c79b60b9 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -313,7 +313,7 @@ namespace kagome::consensus::babe { synchronizer_->syncByBlockHeader( announce.header, peer_id, - [wp = weak_from_this(), announce = announce]( + [wp = weak_from_this(), announce = announce, peer_id]( outcome::result block_res) mutable { if (auto self = wp.lock()) { if (block_res.has_error()) { @@ -322,10 +322,20 @@ namespace kagome::consensus::babe { if (self->current_state_ == Babe::State::CATCHING_UP) { const auto &block = block_res.value(); - SL_INFO(self->log_, "Catching up is finished on block {}", block); - self->current_state_ = Babe::State::SYNCHRONIZED; - self->was_synchronized_ = true; - self->telemetry_->notifyWasSynchronized(); + self->synchronizer_->syncState( + peer_id, + block, + common::Buffer(), + [self](outcome::result + block_res) mutable { + const auto &block = block_res.value(); + SL_INFO(self->log_, + "Catching up is finished on block {}", + block); + self->current_state_ = Babe::State::SYNCHRONIZED; + self->was_synchronized_ = true; + self->telemetry_->notifyWasSynchronized(); + }); } self->onSynchronized(); diff --git a/core/consensus/babe/impl/block_executor_impl.cpp b/core/consensus/babe/impl/block_executor_impl.cpp index a164eb5152..6737653b08 100644 --- a/core/consensus/babe/impl/block_executor_impl.cpp +++ b/core/consensus/babe/impl/block_executor_impl.cpp @@ -100,13 +100,13 @@ namespace kagome::consensus { primitives::BlockInfo block_info(header.number, block_hash); - if (auto body_res = block_tree_->getBlockBody(header.parent_hash); - body_res.has_error() - && body_res.error() == blockchain::BlockTreeError::BODY_NOT_FOUND) { + if (auto header_res = block_tree_->getBlockHeader(header.parent_hash); + header_res.has_error() + && header_res.error() == blockchain::BlockTreeError::HEADER_NOT_FOUND) { logger_->warn("Skipping a block {} with unknown parent", block_info); return Error::PARENT_NOT_FOUND; - } else if (body_res.has_error()) { - return body_res.as_failure(); + } else if (header_res.has_error()) { + return header_res.as_failure(); } // get current time to measure performance if block execution diff --git a/core/network/adapters/protobuf_state_response.hpp b/core/network/adapters/protobuf_state_response.hpp index 1d7e255e1f..be962bca2f 100644 --- a/core/network/adapters/protobuf_state_response.hpp +++ b/core/network/adapters/protobuf_state_response.hpp @@ -50,6 +50,22 @@ namespace kagome::network { if (!msg.ParseFromArray(from.base(), remains)) return AdaptersError::PARSE_FAILED; + for (const auto &kvEntry : msg.entries()) { + KeyValueStateEntry kv; + auto root = kvEntry.state_root(); + if(!root.empty()) { + kv.state_root = storage::trie::RootHash::fromString(root).value(); + } + for (const auto &sEntry : kvEntry.entries()) { + kv.entries.emplace_back(StateEntry{common::Buffer().put(sEntry.key()), + common::Buffer().put(sEntry.value())}); + } + kv.complete = kvEntry.complete(); + + out.entries.emplace_back(std::move(kv)); + } + out.proof = common::Buffer().put(msg.proof()); + std::advance(from, msg.ByteSizeLong()); return from; } diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 05c40d6097..029a5063de 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -11,6 +11,8 @@ #include "blockchain/block_tree_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "primitives/common.hpp" +#include "storage/trie/serialization/trie_serializer.hpp" +#include "storage/trie/trie_storage.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerImpl::Error, e) { using E = kagome::network::SynchronizerImpl::Error; @@ -67,24 +69,31 @@ namespace kagome::network { std::shared_ptr block_tree, std::shared_ptr block_appender, std::shared_ptr block_executor, + std::shared_ptr serializer, + std::shared_ptr storage, std::shared_ptr router, std::shared_ptr scheduler, std::shared_ptr hasher) - : app_config_(app_config), - block_tree_(std::move(block_tree)), + : block_tree_(std::move(block_tree)), block_appender_(std::move(block_appender)), block_executor_(std::move(block_executor)), + serializer_(std::move(serializer)), + storage_(std::move(storage)), router_(std::move(router)), scheduler_(std::move(scheduler)), hasher_(std::move(hasher)) { BOOST_ASSERT(block_tree_); BOOST_ASSERT(block_executor_); + BOOST_ASSERT(serializer_); + BOOST_ASSERT(storage_); BOOST_ASSERT(router_); BOOST_ASSERT(scheduler_); BOOST_ASSERT(hasher_); BOOST_ASSERT(app_state_manager); + sync_method_ = app_config.syncMethod(); + // Register metrics metrics_registry_->registerGaugeFamily( kImportQueueLength, "Number of blocks submitted to the import queue"); @@ -531,7 +540,7 @@ namespace kagome::network { return; } - network::BlocksRequest request{attributesForSync(app_config_.syncMethod()), + network::BlocksRequest request{attributesForSync(sync_method_), from.hash, std::nullopt, network::Direction::ASCENDING, @@ -727,7 +736,9 @@ namespace kagome::network { } SL_TRACE(self->log_, "Block loading is finished"); - if (handler) handler(last_loaded_block); + if(handler) { + handler(last_loaded_block); + } if (some_blocks_added) { SL_TRACE(self->log_, "Enqueued some new blocks: schedule applying"); @@ -860,6 +871,64 @@ namespace kagome::network { protocol->request(peer_id, std::move(request), std::move(response_handler)); } + void SynchronizerImpl::syncState(const libp2p::peer::PeerId &peer_id, + const primitives::BlockInfo& block, + common::Buffer &&key, + SyncResultHandler &&handler) { + if (sync_method_ == application::AppConfiguration::SyncMethod::Fast && not state_syncing_.load()) { + state_syncing_.store(true); + network::StateRequest request{block.hash, {key}, true}; + + auto protocol = router_->getStateProtocol(); + BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); + + SL_TRACE(log_, "State syncing started."); + + auto response_handler = [wp = weak_from_this(), + block, + peer_id, + handler = std::move(handler)]( + auto &&response_res) mutable { + auto self = wp.lock(); + if (not self) { + return; + } + + if (response_res.has_error()) { + SL_WARN(self->log_, + "State syncing failed with error: {}", + response_res.error().message()); + if (handler) handler(response_res.as_failure()); + return; + } + + auto batch = + self->storage_ + ->getPersistentBatchAt(self->serializer_->getEmptyRootHash()) + .value(); + for (const auto &entry : response_res.value().entries[0].entries) { + std::ignore = batch->put(entry.key, entry.value); + } + auto res = batch->commit(); + SL_TRACE(self->log_, + "State syncing finished. Root hash: {}", + res.value().toHex()); + self->sync_method_ = application::AppConfiguration::SyncMethod::Full; + if(handler) { + handler(block); + self->state_syncing_.store(false); + } + }; + + protocol->request( + peer_id, std::move(request), std::move(response_handler)); + } else { + if (handler) { + handler(block); + } + } + } + void SynchronizerImpl::applyNextBlock() { if (generations_.empty()) { SL_TRACE(log_, "No block for applying"); @@ -922,8 +991,7 @@ namespace kagome::network { const BlockInfo block_info(block.header->number, block.hash); auto applying_res = - app_config_.syncMethod() - == application::AppConfiguration::SyncMethod::Full + sync_method_ == application::AppConfiguration::SyncMethod::Full ? block_executor_->applyBlock(std::move(block)) : block_appender_->appendBlock(std::move(block)); diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 04fb28b17a..ec577b5626 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -8,14 +8,15 @@ #include "network/synchronizer.hpp" +#include #include #include #include #include "application/app_state_manager.hpp" -#include "consensus/babe/block_executor.hpp" #include "consensus/babe/block_appender.hpp" +#include "consensus/babe/block_executor.hpp" #include "metrics/metrics.hpp" #include "network/router.hpp" #include "telemetry/service.hpp" @@ -24,6 +25,11 @@ namespace kagome::application { class AppConfiguration; } +namespace kagome::storage::trie { + class TrieSerializer; + class TrieStorage; +} // namespace kagome::storage::trie + namespace kagome::network { class SynchronizerImpl @@ -64,6 +70,8 @@ namespace kagome::network { std::shared_ptr block_tree, std::shared_ptr block_appender, std::shared_ptr block_executor, + std::shared_ptr serializer, + std::shared_ptr storage, std::shared_ptr router, std::shared_ptr scheduler, std::shared_ptr hasher); @@ -92,6 +100,11 @@ namespace kagome::network { std::optional limit, SyncResultHandler &&handler) override; + void syncState(const libp2p::peer::PeerId &peer_id, + const primitives::BlockInfo &block, + common::Buffer &&key, + SyncResultHandler &&handler) override; + /// Finds best common block with peer {@param peer_id} in provided interval. /// It is using tail-recursive algorithm, till {@param hint} is /// the needed block @@ -160,9 +173,12 @@ namespace kagome::network { std::shared_ptr block_tree_; std::shared_ptr block_appender_; std::shared_ptr block_executor_; + std::shared_ptr serializer_; + std::shared_ptr storage_; std::shared_ptr router_; std::shared_ptr scheduler_; std::shared_ptr hasher_; + application::AppConfiguration::SyncMethod sync_method_; // Metrics metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); @@ -171,6 +187,7 @@ namespace kagome::network { log::Logger log_ = log::createLogger("Synchronizer", "synchronizer"); telemetry::Telemetry telemetry_ = telemetry::createTelemetryService(); + std::atomic_bool state_syncing_ = false; bool node_is_shutting_down_ = false; struct KnownBlock { diff --git a/core/network/synchronizer.hpp b/core/network/synchronizer.hpp index b8f9871539..a140ce982c 100644 --- a/core/network/synchronizer.hpp +++ b/core/network/synchronizer.hpp @@ -44,11 +44,15 @@ namespace kagome::network { /// Tries to request block justifications from {@param peer_id} for {@param /// target_block} or a range of blocks up to {@param limit} count. /// Calls {@param handler} when operation finishes - virtual void syncMissingJustifications( - const libp2p::peer::PeerId &peer_id, - primitives::BlockInfo target_block, - std::optional limit, - SyncResultHandler &&handler) = 0; + virtual void syncMissingJustifications(const libp2p::peer::PeerId &peer_id, + primitives::BlockInfo target_block, + std::optional limit, + SyncResultHandler &&handler) = 0; + + virtual void syncState(const libp2p::peer::PeerId &peer_id, + const primitives::BlockInfo &block, + common::Buffer &&key, + SyncResultHandler &&handler) = 0; }; } // namespace kagome::network diff --git a/test/core/consensus/babe/block_executor_test.cpp b/test/core/consensus/babe/block_executor_test.cpp index c9dc09b6c4..9f95d08357 100644 --- a/test/core/consensus/babe/block_executor_test.cpp +++ b/test/core/consensus/babe/block_executor_test.cpp @@ -144,8 +144,6 @@ TEST_F(BlockExecutorTest, JustificationFollowDigests) { .header = header, .body = kagome::primitives::BlockBody{}, .justification = justification}; - EXPECT_CALL(*block_tree_, getBlockBody(BlockId{"parent_hash"_hash256})) - .WillOnce(testing::Return(kagome::primitives::BlockBody{})); EXPECT_CALL(*block_tree_, getBlockBody(BlockId{"some_hash"_hash256})) .WillOnce( testing::Return(kagome::blockchain::BlockTreeError::BODY_NOT_FOUND)); @@ -167,7 +165,7 @@ TEST_F(BlockExecutorTest, JustificationFollowDigests) { "randomness"_hash256)) .WillOnce(testing::Return(outcome::success())); EXPECT_CALL(*block_tree_, getBlockHeader(BlockId{"parent_hash"_hash256})) - .WillOnce(testing::Return(kagome::primitives::BlockHeader{ + .WillRepeatedly(testing::Return(kagome::primitives::BlockHeader{ .parent_hash = "grandparent_hash"_hash256, .number = 40})); EXPECT_CALL(*block_tree_, getLastFinalized()) .WillOnce(testing::Return(BlockInfo{40, "grandparent_hash"_hash256})) diff --git a/test/core/network/synchronizer_test.cpp b/test/core/network/synchronizer_test.cpp index 9ee8f68972..bf2fb93b89 100644 --- a/test/core/network/synchronizer_test.cpp +++ b/test/core/network/synchronizer_test.cpp @@ -17,6 +17,8 @@ #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/network/protocols/sync_protocol_mock.hpp" #include "mock/core/network/router_mock.hpp" +#include "mock/core/storage/trie/serialization/trie_serializer_mock.hpp" +#include "mock/core/storage/trie/trie_storage_mock.hpp" #include "network/impl/synchronizer_impl.hpp" #include "primitives/common.hpp" #include "testutil/literals.hpp" @@ -72,6 +74,8 @@ class SynchronizerTest block_tree, block_appender, block_executor, + serializer, + storage, router, scheduler, hasher); @@ -86,6 +90,10 @@ class SynchronizerTest std::make_shared(); std::shared_ptr block_executor = std::make_shared(); + std::shared_ptr storage = + std::make_shared(); + std::shared_ptr serializer = + std::make_shared(); std::shared_ptr sync_protocol = std::make_shared(); std::shared_ptr router = diff --git a/test/core/network/types/protobuf_state_request_test.cpp b/test/core/network/types/protobuf_state_request_test.cpp index 9f7e72eb44..a7f59dca62 100644 --- a/test/core/network/types/protobuf_state_request_test.cpp +++ b/test/core/network/types/protobuf_state_request_test.cpp @@ -31,9 +31,9 @@ struct ProtobufStateRequestAdapterTest : public ::testing::Test { }; /** - * @given sample `BlocksRequest` instance + * @given sample `StateRequest` instance * @when protobuf serialized into buffer - * @then deserialization `BlocksRequest` from this buffer will contain exactly + * @then deserialization `StateRequest` from this buffer will contain exactly * the same fields with the same values */ TEST_F(ProtobufStateRequestAdapterTest, Serialization) { diff --git a/test/mock/core/host_api/host_api_mock.hpp b/test/mock/core/host_api/host_api_mock.hpp index 722e4eb75f..ea215c889c 100644 --- a/test/mock/core/host_api/host_api_mock.hpp +++ b/test/mock/core/host_api/host_api_mock.hpp @@ -7,6 +7,7 @@ #define KAGOME_TEST_CORE_RUNTIME_MOCK_HOST_API_HPP_ #include "host_api/host_api.hpp" +#include "runtime/types.hpp" #include diff --git a/test/mock/core/network/synchronizer_mock.hpp b/test/mock/core/network/synchronizer_mock.hpp index f91ece9278..a54c548ad5 100644 --- a/test/mock/core/network/synchronizer_mock.hpp +++ b/test/mock/core/network/synchronizer_mock.hpp @@ -53,6 +53,21 @@ namespace kagome::network { SyncResultHandler &&handler) override { return syncMissingJustifications(peer_id, target_block, limit, handler); } + + MOCK_METHOD(void, + syncState, + (const libp2p::peer::PeerId &, + const primitives::BlockInfo &, + const common::Buffer &, + const SyncResultHandler &), + ()); + + void syncState(const libp2p::peer::PeerId &peer_id, + const primitives::BlockInfo &block_info, + common::Buffer &&key, + SyncResultHandler &&handler) override { + return syncState(peer_id, block_info, key, handler); + } }; } // namespace kagome::network From 2eb589348128f39d1ea301250811af4953367d66 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Mon, 25 Apr 2022 15:11:35 +0300 Subject: [PATCH 04/74] recursive fast sync call (#1164) recursive fast sync call (#1164) --- core/blockchain/impl/block_tree_impl.cpp | 37 ++++++++------- core/network/impl/synchronizer_impl.cpp | 48 ++++++++++++-------- core/network/impl/synchronizer_impl.hpp | 6 ++- core/network/synchronizer.hpp | 2 +- core/utils/profiler.hpp | 4 +- test/core/network/CMakeLists.txt | 1 + test/core/network/synchronizer_test.cpp | 15 ++++++ test/mock/core/network/synchronizer_mock.hpp | 2 +- 8 files changed, 75 insertions(+), 40 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 3ef15baf15..db74725a9f 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -824,25 +824,28 @@ namespace kagome::blockchain { chain_events_engine_->notify( primitives::events::ChainEventType::kFinalizedHeads, header); - OUTCOME_TRY(new_runtime_version, runtime_core_->version(block_hash)); - if (not actual_runtime_version_.has_value() - || actual_runtime_version_ != new_runtime_version) { - actual_runtime_version_ = new_runtime_version; - chain_events_engine_->notify( - primitives::events::ChainEventType::kFinalizedRuntimeVersion, - new_runtime_version); + // it has failure result when fast sync is in progress + auto new_runtime_version = runtime_core_->version(block_hash); + if (new_runtime_version.has_value()) { + if (not actual_runtime_version_.has_value() + || actual_runtime_version_ != new_runtime_version.value()) { + actual_runtime_version_ = new_runtime_version.value(); + chain_events_engine_->notify( + primitives::events::ChainEventType::kFinalizedRuntimeVersion, + new_runtime_version.value()); + } } + OUTCOME_TRY(body, storage_->getBlockBody(node->block_hash)); - if (!body.has_value()) { - return BlockTreeError::BODY_NOT_FOUND; - } - for (auto &ext : body.value()) { - if (auto key = - extrinsic_event_key_repo_->get(hasher_->blake2b_256(ext.data))) { - extrinsic_events_engine_->notify( - key.value(), - primitives::events::ExtrinsicLifecycleEvent::Finalized(key.value(), - block_hash)); + if (body.has_value()) { + for (auto &ext : body.value()) { + if (auto key = extrinsic_event_key_repo_->get( + hasher_->blake2b_256(ext.data))) { + extrinsic_events_engine_->notify( + key.value(), + primitives::events::ExtrinsicLifecycleEvent::Finalized( + key.value(), block_hash)); + } } } diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 029a5063de..ad0094db40 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -10,8 +10,10 @@ #include "application/app_configuration.hpp" #include "blockchain/block_tree_error.hpp" #include "network/helpers/peer_id_formatter.hpp" +#include "network/types/block_attributes.hpp" #include "primitives/common.hpp" #include "storage/trie/serialization/trie_serializer.hpp" +#include "storage/trie/trie_batches.hpp" #include "storage/trie/trie_storage.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerImpl::Error, e) { @@ -94,6 +96,9 @@ namespace kagome::network { sync_method_ = app_config.syncMethod(); + batch_ = + storage_->getPersistentBatchAt(serializer_->getEmptyRootHash()).value(); + // Register metrics metrics_registry_->registerGaugeFamily( kImportQueueLength, "Number of blocks submitted to the import queue"); @@ -736,7 +741,7 @@ namespace kagome::network { } SL_TRACE(self->log_, "Block loading is finished"); - if(handler) { + if (handler) { handler(last_loaded_block); } @@ -872,10 +877,11 @@ namespace kagome::network { } void SynchronizerImpl::syncState(const libp2p::peer::PeerId &peer_id, - const primitives::BlockInfo& block, - common::Buffer &&key, + const primitives::BlockInfo &block, + const common::Buffer &key, SyncResultHandler &&handler) { - if (sync_method_ == application::AppConfiguration::SyncMethod::Fast && not state_syncing_.load()) { + if (sync_method_ == application::AppConfiguration::SyncMethod::Fast + && (not state_syncing_.load() || not key.empty())) { state_syncing_.store(true); network::StateRequest request{block.hash, {key}, true}; @@ -902,21 +908,27 @@ namespace kagome::network { return; } - auto batch = - self->storage_ - ->getPersistentBatchAt(self->serializer_->getEmptyRootHash()) - .value(); - for (const auto &entry : response_res.value().entries[0].entries) { - std::ignore = batch->put(entry.key, entry.value); + auto response = response_res.value().entries[0]; + for (const auto &entry : response.entries) { + std::ignore = self->batch_->put(entry.key, entry.value); } - auto res = batch->commit(); - SL_TRACE(self->log_, - "State syncing finished. Root hash: {}", - res.value().toHex()); - self->sync_method_ = application::AppConfiguration::SyncMethod::Full; - if(handler) { - handler(block); - self->state_syncing_.store(false); + self->entries_ += response.entries.size(); + if (response.complete) { + auto res = self->batch_->commit(); + SL_TRACE(self->log_, + "State syncing finished. Root hash: {}", + res.value().toHex()); + self->sync_method_ = application::AppConfiguration::SyncMethod::Full; + if (handler) { + handler(block); + self->state_syncing_.store(false); + } + } else { + SL_TRACE(self->log_, + "State syncing continues. {} entries loaded", + self->entries_); + self->syncState( + peer_id, block, response.entries.back().key, std::move(handler)); } }; diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index ec577b5626..430f1306c5 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -26,6 +26,7 @@ namespace kagome::application { } namespace kagome::storage::trie { + class PersistentTrieBatch; class TrieSerializer; class TrieStorage; } // namespace kagome::storage::trie @@ -102,7 +103,7 @@ namespace kagome::network { void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - common::Buffer &&key, + const common::Buffer &key, SyncResultHandler &&handler) override; /// Finds best common block with peer {@param peer_id} in provided interval. @@ -228,6 +229,9 @@ namespace kagome::network { std::set> recent_requests_; + + std::shared_ptr batch_; + size_t entries_{0}; }; } // namespace kagome::network diff --git a/core/network/synchronizer.hpp b/core/network/synchronizer.hpp index a140ce982c..41aff28372 100644 --- a/core/network/synchronizer.hpp +++ b/core/network/synchronizer.hpp @@ -51,7 +51,7 @@ namespace kagome::network { virtual void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - common::Buffer &&key, + const common::Buffer &key, SyncResultHandler &&handler) = 0; }; diff --git a/core/utils/profiler.hpp b/core/utils/profiler.hpp index a0b76b055c..6f3ba1b62a 100644 --- a/core/utils/profiler.hpp +++ b/core/utils/profiler.hpp @@ -23,9 +23,9 @@ class TicToc { str += "at line " + std::to_string(line); } log_->info( - "{} lasted for {} sec", + "{} lasted for {} microsec", str, - std::chrono::duration_cast(t_ - prev).count()); + std::chrono::duration_cast(t_ - prev).count()); } ~TicToc() { diff --git a/test/core/network/CMakeLists.txt b/test/core/network/CMakeLists.txt index 4f1f616ce9..b8cce9eb09 100644 --- a/test/core/network/CMakeLists.txt +++ b/test/core/network/CMakeLists.txt @@ -59,6 +59,7 @@ target_link_libraries(synchronizer_test logger_for_tests sync_protocol block_tree_error + polkadot_codec p2p::p2p_basic_scheduler p2p::p2p_message_read_writer p2p::p2p_peer_id diff --git a/test/core/network/synchronizer_test.cpp b/test/core/network/synchronizer_test.cpp index bf2fb93b89..e550c9c4a2 100644 --- a/test/core/network/synchronizer_test.cpp +++ b/test/core/network/synchronizer_test.cpp @@ -18,9 +18,11 @@ #include "mock/core/network/protocols/sync_protocol_mock.hpp" #include "mock/core/network/router_mock.hpp" #include "mock/core/storage/trie/serialization/trie_serializer_mock.hpp" +#include "mock/core/storage/trie/trie_batches_mock.hpp" #include "mock/core/storage/trie/trie_storage_mock.hpp" #include "network/impl/synchronizer_impl.hpp" #include "primitives/common.hpp" +#include "storage/trie/serialization/polkadot_codec.hpp" #include "testutil/literals.hpp" #include "testutil/prepare_loggers.hpp" @@ -68,6 +70,19 @@ class SynchronizerTest EXPECT_CALL(*scheduler, scheduleImplMockCall(_, _, _)).Times(AnyNumber()); + EXPECT_CALL(app_config, syncMethod()) + .WillOnce(Return(application::AppConfiguration::SyncMethod::Full)); + + EXPECT_CALL(*serializer, getEmptyRootHash()) + .Times(2) + .WillRepeatedly( + Return(trie::PolkadotCodec().hash256(common::Buffer{0}))); + + outcome::result> batch = + std::make_unique(); + EXPECT_CALL(*storage, getPersistentBatchAt(serializer->getEmptyRootHash())) + .WillOnce(Return(testing::ByMove(std::move(batch)))); + synchronizer = std::make_shared(app_config, app_state_manager, diff --git a/test/mock/core/network/synchronizer_mock.hpp b/test/mock/core/network/synchronizer_mock.hpp index a54c548ad5..bab750289c 100644 --- a/test/mock/core/network/synchronizer_mock.hpp +++ b/test/mock/core/network/synchronizer_mock.hpp @@ -64,7 +64,7 @@ namespace kagome::network { void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block_info, - common::Buffer &&key, + const common::Buffer &key, SyncResultHandler &&handler) override { return syncState(peer_id, block_info, key, handler); } From ebf7fefb58c54326474476f0c58064ea86258ea2 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 27 Apr 2022 21:16:15 +0300 Subject: [PATCH 05/74] add calculation of authorities from origin (#1164) --- .../authority/impl/authority_manager_impl.cpp | 79 ++++++++++++-- .../authority/impl/authority_manager_impl.hpp | 6 ++ core/injector/application_injector.cpp | 38 +++++++ core/network/impl/synchronizer_impl.cpp | 100 +++++++++--------- 4 files changed, 169 insertions(+), 54 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index deaa4ae1db..4752540da9 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -5,6 +5,8 @@ #include "consensus/authority/impl/authority_manager_impl.hpp" +#include +#include #include #include @@ -244,12 +246,19 @@ namespace kagome::authority { significant_block, error.message()); - PREPARE_TRY( - set_id_opt, - fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header), - "Error fetching authority set id from trie storage for block {}: {}", - significant_block, - error.message()); + auto &&set_id_opt_res = + fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header); + if (set_id_opt_res.has_error()) { + auto &error = set_id_opt_res.error(); + log_->warn( + "Couldn't fetch authority set id from trie storage for block #{} " + "({}): {}. Recalculating from genesis.", + header.number, + hash, + error.message()); + return prepareFromGenesis(); + } + auto &set_id_opt = set_id_opt_res.value(); if (not set_id_opt.has_value()) { log_->critical( @@ -840,4 +849,62 @@ namespace kagome::authority { } } + bool AuthorityManagerImpl::prepareFromGenesis() { + auto t_start = std::chrono::high_resolution_clock::now(); + std::vector> + headers; + const auto finalized_block = block_tree_->getLastFinalized(); + auto header = block_tree_->getBlockHeader(finalized_block.hash); + headers.emplace_back(finalized_block.hash, header.value()); + size_t count1 = 0; + while (header.has_value()) { + auto hash = header.value().parent_hash; + header = block_tree_->getBlockHeader(hash); + if (header.has_value()) { + if (not(++count1 % 10000)) { + SL_WARN(log_, "{} headers loaded", count1); + } + headers.emplace_back(hash, header.value()); + } + } + + root_ = authority::ScheduleNode::createAsRoot( + {headers.back().second.number, headers.back().first}); + auto authorities = grandpa_api_->authorities(headers.back().first).value(); + authorities.id = 0; + root_->actual_authorities = + std::make_shared(std::move(authorities)); + + count1 = 0; + size_t count2 = 0; + for (const auto &[hash, header] : boost::range::reverse(headers)) { + if (not(++count1 % 10000)) { + SL_WARN(log_, "{} digests applied ({})", count1, count2); + count2 = 0; + } + for (const auto &digest_item : header.digest) { + std::ignore = visit_in_place( + digest_item, + [&](const primitives::Consensus &consensus_message) + -> outcome::result { + ++count2; + return onConsensus(primitives::BlockInfo{header.number, hash}, + consensus_message); + }, + [](const auto &) { return outcome::success(); }); + } + if (not(count1 % 10000)) { + prune(primitives::BlockInfo{header.number, hash}); + } + } + + auto t_end = std::chrono::high_resolution_clock::now(); + + log_->warn( + "Applied authorities within {} ms", + std::chrono::duration_cast(t_end - t_start) + .count()); + return true; + } + } // namespace kagome::authority diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 44c6c59a35..36da79dd49 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -116,6 +116,12 @@ namespace kagome::authority { void reorganize(std::shared_ptr node, std::shared_ptr new_node); + /** + * @brief Calculates authorities for last finalized block + * starting from genesis + */ + bool prepareFromGenesis(); + Config config_; std::shared_ptr block_tree_; std::shared_ptr trie_storage_; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index c7386be1a0..68d5ba4584 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -985,6 +985,36 @@ namespace { std::forward(args)...); } + template + sptr get_genesis_block_header( + const Injector &injector) { + static auto initialized = + std::optional>(std::nullopt); + if (initialized) { + return initialized.value(); + } + + auto block_storage = + injector.template create>(); + auto block_header_repository = + injector.template create>(); + + auto hash_res = + block_header_repository->getHashByNumber(primitives::BlockNumber(0)); + BOOST_ASSERT(hash_res.has_value()); + auto &hash = hash_res.value(); + + auto header_res = block_storage->getBlockHeader(hash); + BOOST_ASSERT(header_res.has_value()); + auto &header_opt = header_res.value(); + BOOST_ASSERT(header_opt.has_value()); + + initialized.emplace(new primitives::GenesisBlockHeader( + {.header = header_opt.value(), .hash = hash})); + + return initialized.value(); + } + template primitives::BlockHash get_last_finalized_hash(const Injector &injector) { auto storage = injector.template create>(); @@ -1198,6 +1228,14 @@ namespace { di::bind.to([](auto const &injector) { // need it to add genesis block if it's not there auto babe_api = injector.template create>(); + if (injector.template create() + .syncMethod() + == application::AppConfiguration::SyncMethod::Fast) { + auto genesis_block_header = + injector + .template create>(); + return get_babe_configuration(genesis_block_header->hash, babe_api); + } static auto last_finalized_hash = get_last_finalized_hash(injector); return get_babe_configuration(last_finalized_hash, babe_api); }), diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index ad0094db40..2adaed864f 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -880,60 +880,64 @@ namespace kagome::network { const primitives::BlockInfo &block, const common::Buffer &key, SyncResultHandler &&handler) { - if (sync_method_ == application::AppConfiguration::SyncMethod::Fast - && (not state_syncing_.load() || not key.empty())) { - state_syncing_.store(true); - network::StateRequest request{block.hash, {key}, true}; + if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { + if (not state_syncing_.load() || not key.empty()) { + state_syncing_.store(true); + network::StateRequest request{block.hash, {key}, true}; - auto protocol = router_->getStateProtocol(); - BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); + auto protocol = router_->getStateProtocol(); + BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); - SL_TRACE(log_, "State syncing started."); + SL_TRACE(log_, "State syncing started."); - auto response_handler = [wp = weak_from_this(), - block, - peer_id, - handler = std::move(handler)]( - auto &&response_res) mutable { - auto self = wp.lock(); - if (not self) { - return; - } + auto response_handler = + [wp = weak_from_this(), + block, + peer_id, + handler = std::move(handler)](auto &&response_res) mutable { + auto self = wp.lock(); + if (not self) { + return; + } - if (response_res.has_error()) { - SL_WARN(self->log_, - "State syncing failed with error: {}", - response_res.error().message()); - if (handler) handler(response_res.as_failure()); - return; - } + if (response_res.has_error()) { + SL_WARN(self->log_, + "State syncing failed with error: {}", + response_res.error().message()); + if (handler) handler(response_res.as_failure()); + return; + } - auto response = response_res.value().entries[0]; - for (const auto &entry : response.entries) { - std::ignore = self->batch_->put(entry.key, entry.value); - } - self->entries_ += response.entries.size(); - if (response.complete) { - auto res = self->batch_->commit(); - SL_TRACE(self->log_, - "State syncing finished. Root hash: {}", - res.value().toHex()); - self->sync_method_ = application::AppConfiguration::SyncMethod::Full; - if (handler) { - handler(block); - self->state_syncing_.store(false); - } - } else { - SL_TRACE(self->log_, - "State syncing continues. {} entries loaded", - self->entries_); - self->syncState( - peer_id, block, response.entries.back().key, std::move(handler)); - } - }; + auto response = response_res.value().entries[0]; + for (const auto &entry : response.entries) { + std::ignore = self->batch_->put(entry.key, entry.value); + } + self->entries_ += response.entries.size(); + if (response.complete) { + auto res = self->batch_->commit(); + SL_INFO(self->log_, + "State syncing finished. Root hash: {}", + res.value().toHex()); + self->sync_method_ = + application::AppConfiguration::SyncMethod::Full; + if (handler) { + handler(block); + self->state_syncing_.store(false); + } + } else { + SL_TRACE(self->log_, + "State syncing continues. {} entries loaded", + self->entries_); + self->syncState(peer_id, + block, + response.entries.back().key, + std::move(handler)); + } + }; - protocol->request( - peer_id, std::move(request), std::move(response_handler)); + protocol->request( + peer_id, std::move(request), std::move(response_handler)); + } } else { if (handler) { handler(block); From f677f08b5545a54b15fc4c03bdf22656b191951b Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Sat, 28 May 2022 15:23:59 +0300 Subject: [PATCH 06/74] fastsync working on rococo (#1164) --- .../authority/impl/authority_manager_impl.cpp | 10 +- core/consensus/babe/impl/babe_impl.cpp | 33 +- .../grandpa/impl/environment_impl.cpp | 10 +- .../grandpa/impl/environment_impl.hpp | 2 + core/injector/application_injector.cpp | 4 +- core/network/impl/CMakeLists.txt | 2 +- core/network/impl/synchronizer_new_impl.cpp | 656 ++++++++++++++++++ core/network/impl/synchronizer_new_impl.hpp | 180 +++++ core/network/synchronizer.hpp | 2 + core/network/types/grandpa_message.hpp | 21 + 10 files changed, 900 insertions(+), 20 deletions(-) create mode 100644 core/network/impl/synchronizer_new_impl.cpp create mode 100644 core/network/impl/synchronizer_new_impl.hpp diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 4752540da9..2eb0f8431e 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -699,10 +699,10 @@ namespace kagome::authority { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; }); } else { - SL_WARN(log_, - "Unknown consensus engine id in block {}: {}", - block, - message.consensus_engine_id.toString()); + // SL_WARN(log_, + // "Unknown consensus engine id in block {}: {}", + // block, + // message.consensus_engine_id.toString()); return outcome::success(); } } @@ -878,7 +878,7 @@ namespace kagome::authority { count1 = 0; size_t count2 = 0; for (const auto &[hash, header] : boost::range::reverse(headers)) { - if (not(++count1 % 10000)) { + if (not(++count1 % 1000)) { SL_WARN(log_, "{} digests applied ({})", count1, count2); count2 = 0; } diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 79c79b60b9..ecf1805e39 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -316,6 +316,8 @@ namespace kagome::consensus::babe { [wp = weak_from_this(), announce = announce, peer_id]( outcome::result block_res) mutable { if (auto self = wp.lock()) { + self->synchronizer_->endSync(); + if (block_res.has_error()) { return; } @@ -326,8 +328,12 @@ namespace kagome::consensus::babe { peer_id, block, common::Buffer(), - [self](outcome::result - block_res) mutable { + [self, block = block, peer_id]( + outcome::result + block_res) mutable { + if (block_res.has_error()) { + return; + } const auto &block = block_res.value(); SL_INFO(self->log_, "Catching up is finished on block {}", @@ -353,27 +359,34 @@ namespace kagome::consensus::babe { auto is_ran = synchronizer_->syncByBlockInfo( target_block, peer_id, - [wp = weak_from_this(), - bn = target_block.number](outcome::result res) { + [wp = weak_from_this(), bn = target_block.number, peer_id]( + outcome::result res) { if (auto self = wp.lock()) { + self->synchronizer_->endSync(); if (res.has_error()) { SL_DEBUG(self->log_, - "Catching up to block #{} is failed: {}", + "Catching up to block #{} on {} is failed: {}", bn, + peer_id.toBase58(), res.error().message()); return; } - SL_DEBUG(self->log_, - "Catching up to block #{} is going (on block #{} now)", - bn, - res.value().number); + SL_DEBUG( + self->log_, + "Catching up to block #{} on {} is going (on block #{} now)", + bn, + peer_id.toBase58(), + res.value().number); } }, false); if (is_ran) { - SL_VERBOSE(log_, "Catching up to block #{} is ran", target_block.number); + SL_VERBOSE(log_, + "Catching up to block #{} on {} is ran", + target_block.number, + peer_id.toBase58()); current_state_ = State::CATCHING_UP; } } diff --git a/core/consensus/grandpa/impl/environment_impl.cpp b/core/consensus/grandpa/impl/environment_impl.cpp index e50eaa28ff..d4370b2b43 100644 --- a/core/consensus/grandpa/impl/environment_impl.cpp +++ b/core/consensus/grandpa/impl/environment_impl.cpp @@ -94,10 +94,16 @@ namespace kagome::consensus::grandpa { const libp2p::peer::PeerId &peer_id, MembershipCounter set_id, RoundNumber round_number) { - SL_DEBUG( - logger_, "Send Catch-Up-Request beginning with round {}", round_number); network::CatchUpRequest message{.round_number = round_number, .voter_set_id = set_id}; + if (not recent_catchup_requests_.emplace(message.fingerprint()).second) { + SL_ERROR(logger_, + "Don't Send Duplicate Catch-Up-Request beginning with round {}", + round_number); + return outcome::success(); + } + SL_DEBUG( + logger_, "Send Catch-Up-Request beginning with round {}", round_number); transmitter_->sendCatchUpRequest(peer_id, std::move(message)); return outcome::success(); } diff --git a/core/consensus/grandpa/impl/environment_impl.hpp b/core/consensus/grandpa/impl/environment_impl.hpp index 1b50cd8bf9..eeca0c8e98 100644 --- a/core/consensus/grandpa/impl/environment_impl.hpp +++ b/core/consensus/grandpa/impl/environment_impl.hpp @@ -105,6 +105,8 @@ namespace kagome::consensus::grandpa { std::shared_ptr transmitter_; std::weak_ptr justification_observer_; + std::set recent_catchup_requests_; + log::Logger logger_; }; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 68d5ba4584..14764d841c 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -94,7 +94,7 @@ #include "network/impl/router_libp2p.hpp" #include "network/impl/state_protocol_observer_impl.hpp" #include "network/impl/sync_protocol_observer_impl.hpp" -#include "network/impl/synchronizer_impl.hpp" +#include "network/impl/synchronizer_new_impl.hpp" #include "network/impl/transactions_transmitter_impl.hpp" #include "network/sync_protocol_observer.hpp" #include "offchain/impl/offchain_local_storage.hpp" @@ -1239,7 +1239,7 @@ namespace { static auto last_finalized_hash = get_last_finalized_hash(injector); return get_babe_configuration(last_finalized_hash, babe_api); }), - di::bind.template to(), + di::bind.template to(), di::bind.template to(), di::bind.template to(), di::bind.template to(), diff --git a/core/network/impl/CMakeLists.txt b/core/network/impl/CMakeLists.txt index 9d69849cfe..bd120be17f 100644 --- a/core/network/impl/CMakeLists.txt +++ b/core/network/impl/CMakeLists.txt @@ -6,7 +6,7 @@ add_subdirectory(protocols) add_library(synchronizer - synchronizer_impl.cpp + synchronizer_new_impl.cpp ) target_link_libraries(synchronizer logger diff --git a/core/network/impl/synchronizer_new_impl.cpp b/core/network/impl/synchronizer_new_impl.cpp new file mode 100644 index 0000000000..7a648258d1 --- /dev/null +++ b/core/network/impl/synchronizer_new_impl.cpp @@ -0,0 +1,656 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "network/impl/synchronizer_new_impl.hpp" + +#include + +#include "application/app_configuration.hpp" +#include "blockchain/block_tree_error.hpp" +#include "network/helpers/peer_id_formatter.hpp" +#include "network/types/block_attributes.hpp" +#include "primitives/common.hpp" +#include "storage/trie/serialization/trie_serializer.hpp" +#include "storage/trie/trie_batches.hpp" +#include "storage/trie/trie_storage.hpp" + +OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerNewImpl::Error, e) { + using E = kagome::network::SynchronizerNewImpl::Error; + switch (e) { + case E::SHUTTING_DOWN: + return "Node is shutting down"; + case E::EMPTY_RESPONSE: + return "Response is empty"; + case E::RESPONSE_WITHOUT_BLOCK_HEADER: + return "Response does not contain header of some block"; + case E::RESPONSE_WITHOUT_BLOCK_BODY: + return "Response does not contain body of some block"; + case E::DISCARDED_BLOCK: + return "Block is discarded"; + case E::WRONG_ORDER: + return "Wrong order of blocks/headers in response"; + case E::INVALID_HASH: + return "Hash does not match"; + case E::ALREADY_IN_QUEUE: + return "Block is already enqueued"; + case E::PEER_BUSY: + return "Peer is busy"; + case E::ARRIVED_TOO_EARLY: + return "Block is arrived too early. Try to process it late"; + case E::DUPLICATE_REQUEST: + return "Duplicate of recent request has been detected"; + } + return "unknown error"; +} + +namespace { + constexpr const char *kImportQueueLength = + "kagome_import_queue_blocks_submitted"; + + kagome::network::BlockAttributes attributesForSync( + kagome::application::AppConfiguration::SyncMethod method) { + using SM = kagome::application::AppConfiguration::SyncMethod; + switch (method) { + case SM::Full: + return kagome::network::BlocksRequest::kBasicAttributes; + case SM::Fast: + return kagome::network::BlockAttribute::HEADER + | kagome::network::BlockAttribute::JUSTIFICATION; + } + return kagome::network::BlocksRequest::kBasicAttributes; + } +} // namespace + +namespace kagome::network { + + SynchronizerNewImpl::SynchronizerNewImpl( + const application::AppConfiguration &app_config, + std::shared_ptr app_state_manager, + std::shared_ptr block_tree, + std::shared_ptr block_appender, + std::shared_ptr block_executor, + std::shared_ptr serializer, + std::shared_ptr storage, + std::shared_ptr router, + std::shared_ptr scheduler, + std::shared_ptr hasher) + : block_tree_(std::move(block_tree)), + block_appender_(std::move(block_appender)), + block_executor_(std::move(block_executor)), + serializer_(std::move(serializer)), + storage_(std::move(storage)), + router_(std::move(router)), + scheduler_(std::move(scheduler)), + hasher_(std::move(hasher)) { + BOOST_ASSERT(block_tree_); + BOOST_ASSERT(block_executor_); + BOOST_ASSERT(serializer_); + BOOST_ASSERT(storage_); + BOOST_ASSERT(router_); + BOOST_ASSERT(scheduler_); + BOOST_ASSERT(hasher_); + + BOOST_ASSERT(app_state_manager); + + sync_method_ = app_config.syncMethod(); + + batch_ = + storage_->getPersistentBatchAt(serializer_->getEmptyRootHash()).value(); + + // Register metrics + metrics_registry_->registerGaugeFamily( + kImportQueueLength, "Number of blocks submitted to the import queue"); + metric_import_queue_length_ = + metrics_registry_->registerGaugeMetric(kImportQueueLength); + metric_import_queue_length_->set(0); + + app_state_manager->atShutdown([this] { node_is_shutting_down_ = true; }); + } + + bool SynchronizerNewImpl::subscribeToBlock( + const primitives::BlockInfo &block_info, SyncResultHandler &&handler) { + // Check if block is already in tree + if (block_tree_->hasBlockHeader(block_info.hash)) { + scheduler_->schedule( + [handler = std::move(handler), block_info] { handler(block_info); }); + return false; + } + + auto last_finalized_block = block_tree_->getLastFinalized(); + // Check if block from discarded side-chain + if (last_finalized_block.number <= block_info.number) { + scheduler_->schedule( + [handler = std::move(handler)] { handler(Error::DISCARDED_BLOCK); }); + return false; + } + + // Check if block has arrived too early + auto best_block_res = + block_tree_->getBestContaining(last_finalized_block.hash, std::nullopt); + BOOST_ASSERT(best_block_res.has_value()); + const auto &best_block = best_block_res.value(); + if (best_block.number + kMaxDistanceToBlockForSubscription + < block_info.number) { + scheduler_->schedule([handler = std::move(handler)] { + handler(Error::ARRIVED_TOO_EARLY); + }); + return false; + } + + subscriptions_.emplace(block_info, std::move(handler)); + return true; + } + + void SynchronizerNewImpl::notifySubscribers( + const primitives::BlockInfo &block, outcome::result res) { + auto range = subscriptions_.equal_range(block); + for (auto it = range.first; it != range.second;) { + auto cit = it++; + if (auto node = subscriptions_.extract(cit)) { + if (res.has_error()) { + auto error = res.as_failure(); + scheduler_->schedule( + [handler = std::move(node.mapped()), error] { handler(error); }); + } else { + scheduler_->schedule( + [handler = std::move(node.mapped()), block] { handler(block); }); + } + } + } + } + + bool SynchronizerNewImpl::syncByBlockInfo( + const primitives::BlockInfo &block_info, + const libp2p::peer::PeerId &peer_id, + Synchronizer::SyncResultHandler &&handler, + bool subscribe_to_block) { + if (not state_syncing_.load() && not syncing_.load()) { + syncing_.store(true); + } else { + return false; + } + // Subscribe on demand + if (subscribe_to_block) { + subscribeToBlock(block_info, std::move(handler)); + } + + // If provided block is already enqueued, just remember peer + if (auto it = known_blocks_.find(block_info.hash); + it != known_blocks_.end()) { + auto &block_in_queue = it->second; + block_in_queue.peers.emplace(peer_id); + if (handler) handler(block_info); + return false; + } + + const auto &last_finalized_block = block_tree_->getLastFinalized(); + + auto best_block_res = + block_tree_->getBestContaining(last_finalized_block.hash, std::nullopt); + BOOST_ASSERT(best_block_res.has_value()); + const auto &best_block = best_block_res.value(); + + // Provided block is equal our best one. Nothing needs to do. + if (block_info == best_block) { + if (handler) handler(block_info); + return false; + } + + loadBlocks(peer_id, last_finalized_block, std::move(handler)); + + // scheduler_->schedule([wp = weak_from_this()] { + // if (auto self = wp.lock()) { + // self->applyNextBlock(); + // } + // }); + + return true; + } + + bool SynchronizerNewImpl::syncByBlockHeader( + const primitives::BlockHeader &header, + const libp2p::peer::PeerId &peer_id, + Synchronizer::SyncResultHandler &&handler) { + if (not state_syncing_.load() && not syncing_.load()) { + syncing_.store(true); + } else { + return false; + } + auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); + const primitives::BlockInfo block_info(header.number, block_hash); + + // Block was applied before + if (block_tree_->getBlockHeader(block_hash).has_value()) { + return false; + } + + // Block is already enqueued + if (auto it = known_blocks_.find(block_info.hash); + it != known_blocks_.end()) { + auto &block_in_queue = it->second; + block_in_queue.peers.emplace(peer_id); + return false; + } + + loadBlocks(peer_id, block_info, std::move(handler)); + + // scheduler_->schedule([wp = weak_from_this()] { + // if (auto self = wp.lock()) { + // self->applyNextBlock(); + // } + // }); + return true; + } + + void SynchronizerNewImpl::loadBlocks(const libp2p::peer::PeerId &peer_id, + primitives::BlockInfo from, + SyncResultHandler &&handler) { + // Interrupts process if node is shutting down + if (node_is_shutting_down_) { + if (handler) handler(Error::SHUTTING_DOWN); + return; + } + + network::BlocksRequest request{attributesForSync(sync_method_), + from.hash, + std::nullopt, + network::Direction::ASCENDING, + std::nullopt}; + + auto response_handler = [wp = weak_from_this(), + from, + peer_id, + handler = std::move(handler), + parent_hash = primitives::BlockHash{}]( + auto &&response_res) mutable { + auto self = wp.lock(); + if (not self) { + return; + } + + // Any error interrupts loading of blocks + if (response_res.has_error()) { + SL_ERROR(self->log_, + "Can't load blocks from {} beginning block {}: {}", + peer_id, + from, + response_res.error().message()); + if (handler) handler(response_res.as_failure()); + return; + } + auto &blocks = response_res.value().blocks; + + // No block in response is abnormal situation. + // At least one starting block should be returned as existing + if (blocks.empty()) { + SL_ERROR(self->log_, + "Can't load blocks from {} beginning block {}: " + "Response does not have any blocks", + peer_id, + from); + if (handler) handler(Error::EMPTY_RESPONSE); + return; + } + + SL_TRACE(self->log_, + "{} blocks are loaded from {} beginning block {}", + blocks.size(), + peer_id, + from); + + primitives::BlockInfo last_loaded_block; + + for (auto &block : blocks) { + // Check if header is provided + if (not block.header.has_value()) { + SL_ERROR(self->log_, + "Can't load blocks from {} starting from block {}: " + "Received block without header", + peer_id, + from); + if (handler) handler(Error::RESPONSE_WITHOUT_BLOCK_HEADER); + return; + } + // Check if body is provided + if (not block.header.has_value()) { + SL_ERROR(self->log_, + "Can't load blocks from {} starting from block {}: " + "Received block without body", + peer_id, + from); + if (handler) handler(Error::RESPONSE_WITHOUT_BLOCK_BODY); + return; + } + auto &header = block.header.value(); + + const auto &last_finalized_block = + self->block_tree_->getLastFinalized(); + + // Check by number if block is not finalized yet + if (last_finalized_block.number >= header.number) { + if (last_finalized_block.number == header.number) { + if (last_finalized_block.hash != block.hash) { + SL_ERROR(self->log_, + "Can't load blocks from {} starting from block {}: " + "Received discarded block {}", + peer_id, + from, + BlockInfo(header.number, block.hash)); + if (handler) handler(Error::DISCARDED_BLOCK); + return; + } + + SL_TRACE(self->log_, + "Skip block {} received from {}: " + "it is finalized with block #{}", + BlockInfo(header.number, block.hash), + peer_id, + last_finalized_block.number); + continue; + } + + SL_TRACE(self->log_, + "Skip block {} received from {}: " + "it is below the last finalized block #{}", + BlockInfo(header.number, block.hash), + peer_id, + last_finalized_block.number); + continue; + } + + // Check if block is not discarded + if (last_finalized_block.number + 1 == header.number) { + if (last_finalized_block.hash != header.parent_hash) { + SL_ERROR(self->log_, + "Can't complete blocks loading from {} starting from " + "block {}: Received discarded block {}", + peer_id, + from, + BlockInfo(header.number, header.parent_hash)); + if (handler) handler(Error::DISCARDED_BLOCK); + return; + } + + // Start to check parents + parent_hash = header.parent_hash; + } + + // Check if block is in chain + static const primitives::BlockHash zero_hash; + if (parent_hash != header.parent_hash && parent_hash != zero_hash) { + SL_ERROR(self->log_, + "Can't complete blocks loading from {} starting from " + "block {}: Received block is not descendant of previous", + peer_id, + from); + if (handler) handler(Error::WRONG_ORDER); + return; + } + + // Check if hash is valid + auto calculated_hash = + self->hasher_->blake2b_256(scale::encode(header).value()); + if (block.hash != calculated_hash) { + SL_ERROR(self->log_, + "Can't complete blocks loading from {} starting from " + "block {}: " + "Received block whose hash does not match the header", + peer_id, + from); + if (handler) handler(Error::INVALID_HASH); + return; + } + + last_loaded_block = {header.number, block.hash}; + + parent_hash = block.hash; + + // Add block in queue and save peer or just add peer for existing record + auto it = self->known_blocks_.find(block.hash); + if (it == self->known_blocks_.end()) { + self->known_blocks_.emplace(block.hash, KnownBlock{block, {peer_id}}); + self->metric_import_queue_length_->set(self->known_blocks_.size()); + } else { + it->second.peers.emplace(peer_id); + SL_TRACE(self->log_, + "Skip block {} received from {}: already enqueued", + BlockInfo(header.number, block.hash), + peer_id); + continue; + } + + SL_TRACE(self->log_, + "Enqueue block {} received from {}", + BlockInfo(header.number, block.hash), + peer_id); + + self->generations_.emplace(header.number, block.hash); + self->ancestry_.emplace(header.parent_hash, block.hash); + } + + // handler(last_loaded_block); + + if (blocks.size() < 128) { + self->applyNextBlock(); + handler(last_loaded_block); + return; + } + self->scheduler_->schedule([self, + peer_id, + handler = std::move(handler), + last_loaded_block]() mutable { + self->applyNextBlock(); + self->loadBlocks(peer_id, last_loaded_block, std::move(handler)); + }); + }; + + auto protocol = router_->getSyncProtocol(); + BOOST_ASSERT_MSG(protocol, "Router did not provide sync protocol"); + protocol->request(peer_id, std::move(request), std::move(response_handler)); + } + + void SynchronizerNewImpl::syncState(const libp2p::peer::PeerId &peer_id, + const primitives::BlockInfo &block, + const common::Buffer &key, + SyncResultHandler &&handler) { + if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { + if (not state_syncing_.load() || not key.empty()) { + state_syncing_.store(true); + if (key.empty()) { + sync_block_ = block; + } + network::StateRequest request{block.hash, {key}, true}; + + auto protocol = router_->getStateProtocol(); + BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); + + SL_TRACE(log_, "State syncing started."); + + auto response_handler = + [wp = weak_from_this(), + block, + peer_id, + handler = std::move(handler)](auto &&response_res) mutable { + auto self = wp.lock(); + if (not self) { + return; + } + + if (response_res.has_error()) { + SL_WARN(self->log_, + "State syncing failed with error: {}", + response_res.error().message()); + if (handler) handler(response_res.as_failure()); + return; + } + + auto response = response_res.value().entries[0]; + for (const auto &entry : response.entries) { + std::ignore = self->batch_->put(entry.key, entry.value); + } + self->entries_ += response.entries.size(); + if (response.complete) { + auto res = self->batch_->commit(); + SL_INFO(self->log_, + "State syncing finished. Root hash: {}", + res.value().toHex()); + self->sync_method_ = + application::AppConfiguration::SyncMethod::Full; + if (handler) { + handler(block); + self->state_syncing_.store(false); + } + } else { + SL_TRACE(self->log_, + "State syncing continues. {} entries loaded", + self->entries_); + self->syncState(peer_id, + block, + response.entries.back().key, + std::move(handler)); + } + }; + + protocol->request( + peer_id, std::move(request), std::move(response_handler)); + } + } else { + if (handler) { + handler(block); + } + } + } + + void SynchronizerNewImpl::applyNextBlock() { + if (generations_.empty()) { + SL_TRACE(log_, "No block for applying"); + return; + } + + bool false_val = false; + // if (not applying_in_progress_.compare_exchange_strong(false_val, true)) { + // SL_TRACE(log_, "Applying in progress"); + // return; + // } + // SL_TRACE(log_, "Begin applying"); + // auto cleanup = gsl::finally([this] { + // SL_TRACE(log_, "End applying"); + // applying_in_progress_ = false; + // }); + + primitives::BlockHash hash; + + while (true) { + auto generation_node = generations_.extract(generations_.begin()); + if (generation_node) { + hash = generation_node.mapped(); + break; + } + if (generations_.empty()) { + SL_TRACE(log_, "No block for applying"); + return; + } + } + + auto node = known_blocks_.extract(hash); + if (node) { + auto &block = node.mapped().data; + BOOST_ASSERT(block.header.has_value()); + auto number = block.header->number; + + const auto &last_finalized_block = block_tree_->getLastFinalized(); + + if (block.header->number <= last_finalized_block.number) { + auto header_res = block_tree_->getBlockHeader(hash); + if (not header_res.has_value()) { + auto n = discardBlock(block.hash); + SL_WARN( + log_, + "Block {} {} not applied as discarded", + BlockInfo(number, hash), + n ? fmt::format("and {} others have", n) : fmt::format("has")); + } + } else { + const BlockInfo block_info(block.header->number, block.hash); + + if (sync_method_ == application::AppConfiguration::SyncMethod::Full + && sync_block_ && block_info.number <= sync_block_.value().number) { + applyNextBlock(); + } else { + if (sync_method_ == application::AppConfiguration::SyncMethod::Full + && sync_block_) { + sync_block_ = std::nullopt; + } + auto applying_res = + sync_method_ == application::AppConfiguration::SyncMethod::Full + ? block_executor_->applyBlock(std::move(block)) + : block_appender_->appendBlock(std::move(block)); + + notifySubscribers({number, hash}, applying_res); + + if (not applying_res.has_value()) { + if (applying_res + != outcome::failure(blockchain::BlockTreeError::BLOCK_EXISTS)) { + notifySubscribers({number, hash}, applying_res.as_failure()); + auto n = discardBlock(block_info.hash); + SL_WARN( + log_, + "Block {} {} been discarded: {}", + block_info, + n ? fmt::format("and {} others have", n) : fmt::format("has"), + applying_res.error().message()); + } else { + SL_DEBUG(log_, "Block {} is skipped as existing", block_info); + applyNextBlock(); + } + } else { + applyNextBlock(); + } + } + } + } + ancestry_.erase(hash); + + metric_import_queue_length_->set(known_blocks_.size()); + + // scheduler_->schedule([wp = weak_from_this()] { + // if (auto self = wp.lock()) { + // self->applyNextBlock(); + // } + // }); + } + + size_t SynchronizerNewImpl::discardBlock( + const primitives::BlockHash &hash_of_discarding_block) { + std::queue queue; + queue.emplace(hash_of_discarding_block); + + size_t affected = 0; + while (not queue.empty()) { + const auto &hash = queue.front(); + + if (auto it = known_blocks_.find(hash); it != known_blocks_.end()) { + auto number = it->second.data.header->number; + notifySubscribers({number, hash}, Error::DISCARDED_BLOCK); + + known_blocks_.erase(it); + affected++; + } + + auto range = ancestry_.equal_range(hash); + for (auto it = range.first; it != range.second; ++it) { + queue.emplace(it->second); + } + ancestry_.erase(range.first, range.second); + + queue.pop(); + } + + metric_import_queue_length_->set(known_blocks_.size()); + + return affected; + } + +} // namespace kagome::network diff --git a/core/network/impl/synchronizer_new_impl.hpp b/core/network/impl/synchronizer_new_impl.hpp new file mode 100644 index 0000000000..1a244768e6 --- /dev/null +++ b/core/network/impl/synchronizer_new_impl.hpp @@ -0,0 +1,180 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_NETWORK_SYNCHRONIZERNEWIMPL +#define KAGOME_NETWORK_SYNCHRONIZERNEWIMPL + +#include "network/synchronizer.hpp" + +#include +#include + +#include + +#include "application/app_state_manager.hpp" +#include "consensus/babe/block_appender.hpp" +#include "consensus/babe/block_executor.hpp" +#include "metrics/metrics.hpp" +#include "network/router.hpp" + +namespace kagome::application { + class AppConfiguration; +} + +namespace kagome::storage::trie { + class PersistentTrieBatch; + class TrieSerializer; + class TrieStorage; +} // namespace kagome::storage::trie + +namespace kagome::network { + + class SynchronizerNewImpl + : public Synchronizer, + public std::enable_shared_from_this { + public: + /// Block amount enough for applying and preloading other ones + /// simultaneously. + /// 256 is doubled max amount block in BlocksResponse. + static constexpr size_t kMinPreloadedBlockAmount = 256; + + /// Indicating how far the block can be subscribed to. + /// In general we don't needed wait very far blocks. This limit to avoid + /// extra memory consumption. + static constexpr size_t kMaxDistanceToBlockForSubscription = + kMinPreloadedBlockAmount * 2; + + static constexpr std::chrono::milliseconds kRecentnessDuration = + std::chrono::seconds(60); + + enum class Error { + SHUTTING_DOWN = 1, + EMPTY_RESPONSE, + RESPONSE_WITHOUT_BLOCK_HEADER, + RESPONSE_WITHOUT_BLOCK_BODY, + DISCARDED_BLOCK, + WRONG_ORDER, + INVALID_HASH, + ALREADY_IN_QUEUE, + PEER_BUSY, + ARRIVED_TOO_EARLY, + DUPLICATE_REQUEST + }; + + SynchronizerNewImpl( + const application::AppConfiguration &app_config, + std::shared_ptr app_state_manager, + std::shared_ptr block_tree, + std::shared_ptr block_appender, + std::shared_ptr block_executor, + std::shared_ptr serializer, + std::shared_ptr storage, + std::shared_ptr router, + std::shared_ptr scheduler, + std::shared_ptr hasher); + + /// Enqueues loading (and applying) blocks from peer {@param peer_id} + /// since best common block up to provided {@param block_info}. + /// {@param handler} will be called when this process is finished or failed + /// @returns true if sync is ran (peer is not busy) + /// @note Is used for start/continue catching up. + bool syncByBlockInfo(const primitives::BlockInfo &block_info, + const libp2p::peer::PeerId &peer_id, + SyncResultHandler &&handler, + bool subscribe_to_block) override; + + /// Enqueues loading and applying block {@param block_info} from peer + /// {@param peer_id}. + /// @returns true if sync is ran (peer is not busy) + /// If provided block is the best after applying, {@param handler} be called + bool syncByBlockHeader(const primitives::BlockHeader &header, + const libp2p::peer::PeerId &peer_id, + SyncResultHandler &&handler) override; + + void syncState(const libp2p::peer::PeerId &peer_id, + const primitives::BlockInfo &block, + const common::Buffer &key, + SyncResultHandler &&handler) override; + + /// Loads blocks from peer {@param peer_id} since block {@param from} till + /// its best. Calls {@param handler} when process is finished or failed + void loadBlocks(const libp2p::peer::PeerId &peer_id, + primitives::BlockInfo from, + SyncResultHandler &&handler); + + void endSync() override { + syncing_.store(false); + } + + private: + /// Subscribes handler for block with provided {@param block_info} + /// {@param handler} will be called When block is received or discarded + /// @returns true if subscription is successful + bool subscribeToBlock(const primitives::BlockInfo &block_info, + SyncResultHandler &&handler); + + /// Notifies subscribers about arrived block + void notifySubscribers(const primitives::BlockInfo &block_info, + outcome::result res); + + /// Pops next block from queue and tries to apply that + void applyNextBlock(); + + /// Removes block {@param block} and all all dependent on it from the queue + /// @returns number of affected blocks + size_t discardBlock(const primitives::BlockHash &block); + + std::shared_ptr block_tree_; + std::shared_ptr block_appender_; + std::shared_ptr block_executor_; + std::shared_ptr serializer_; + std::shared_ptr storage_; + std::shared_ptr router_; + std::shared_ptr scheduler_; + std::shared_ptr hasher_; + application::AppConfiguration::SyncMethod sync_method_; + + // Metrics + metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); + metrics::Gauge *metric_import_queue_length_; + + log::Logger log_ = log::createLogger("Synchronizer", "synchronizer"); + + std::atomic_bool syncing_ = false; + std::atomic_bool state_syncing_ = false; + bool node_is_shutting_down_ = false; + + struct KnownBlock { + /// Data of block + primitives::BlockData data; + /// Peers who know this block + std::set peers; + }; + + // Already known (enqueued) but is not applied yet + std::unordered_map known_blocks_; + + std::optional sync_block_; + + // Blocks grouped by number + std::multimap generations_; + + // Links parent->child + std::unordered_multimap + ancestry_; + + std::multimap subscriptions_; + + std::atomic_bool applying_in_progress_ = false; + + std::shared_ptr batch_; + size_t entries_{0}; + }; + +} // namespace kagome::network + +OUTCOME_HPP_DECLARE_ERROR(kagome::network, SynchronizerNewImpl::Error) + +#endif // KAGOME_NETWORK_SYNCHRONIZERNEWIMPL diff --git a/core/network/synchronizer.hpp b/core/network/synchronizer.hpp index 41aff28372..ef15c9fdea 100644 --- a/core/network/synchronizer.hpp +++ b/core/network/synchronizer.hpp @@ -53,6 +53,8 @@ namespace kagome::network { const primitives::BlockInfo &block, const common::Buffer &key, SyncResultHandler &&handler) = 0; + + virtual void endSync() = 0; }; } // namespace kagome::network diff --git a/core/network/types/grandpa_message.hpp b/core/network/types/grandpa_message.hpp index 4a3cf787fe..a4d9d32509 100644 --- a/core/network/types/grandpa_message.hpp +++ b/core/network/types/grandpa_message.hpp @@ -138,4 +138,25 @@ namespace kagome::network { } // namespace kagome::network +template <> +struct std::hash { + auto operator()( + const kagome::network::CatchUpRequest &catch_up_request) const { + auto result = std::hash()( + catch_up_request.round_number); + + boost::hash_combine( + result, + std::hash()( + catch_up_request.voter_set_id)); + + return result; + } +}; + +inline kagome::network::CatchUpRequest::Fingerprint +kagome::network::CatchUpRequest::fingerprint() const { + return std::hash()(*this); +} + #endif // KAGOME_NETWORK_GRANDPAMESSAGE From b2c762894fe8c89fbf3b4bb07e01879a1c5d1666 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Fri, 3 Jun 2022 10:11:36 +0300 Subject: [PATCH 07/74] implement child state request (#1164) --- core/consensus/babe/impl/babe_impl.cpp | 19 ++- core/network/impl/synchronizer_new_impl.cpp | 179 ++++++++++---------- core/network/impl/synchronizer_new_impl.hpp | 9 +- core/network/synchronizer.hpp | 2 +- 4 files changed, 112 insertions(+), 97 deletions(-) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index ecf1805e39..85d794648a 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -299,13 +299,15 @@ namespace kagome::consensus::babe { } // Start catching up if gap recognized - if (announce.header.number > current_best_block.number + 1) { - auto block_hash = - hasher_->blake2b_256(scale::encode(announce.header).value()); - const primitives::BlockInfo announced_block(announce.header.number, - block_hash); - startCatchUp(peer_id, announced_block); - return; + if (current_state_ == Babe::State::SYNCHRONIZED) { + if (announce.header.number > current_best_block.number + 1) { + auto block_hash = + hasher_->blake2b_256(scale::encode(announce.header).value()); + const primitives::BlockInfo announced_block(announce.header.number, + block_hash); + startCatchUp(peer_id, announced_block); + return; + } } // Received announce that has the same block number as ours best, @@ -327,13 +329,14 @@ namespace kagome::consensus::babe { self->synchronizer_->syncState( peer_id, block, - common::Buffer(), + {common::Buffer()}, [self, block = block, peer_id]( outcome::result block_res) mutable { if (block_res.has_error()) { return; } + const auto &block = block_res.value(); SL_INFO(self->log_, "Catching up is finished on block {}", diff --git a/core/network/impl/synchronizer_new_impl.cpp b/core/network/impl/synchronizer_new_impl.cpp index 7a648258d1..e8bdd573e7 100644 --- a/core/network/impl/synchronizer_new_impl.cpp +++ b/core/network/impl/synchronizer_new_impl.cpp @@ -96,9 +96,6 @@ namespace kagome::network { sync_method_ = app_config.syncMethod(); - batch_ = - storage_->getPersistentBatchAt(serializer_->getEmptyRootHash()).value(); - // Register metrics metrics_registry_->registerGaugeFamily( kImportQueueLength, "Number of blocks submitted to the import queue"); @@ -200,12 +197,6 @@ namespace kagome::network { loadBlocks(peer_id, last_finalized_block, std::move(handler)); - // scheduler_->schedule([wp = weak_from_this()] { - // if (auto self = wp.lock()) { - // self->applyNextBlock(); - // } - // }); - return true; } @@ -236,11 +227,6 @@ namespace kagome::network { loadBlocks(peer_id, block_info, std::move(handler)); - // scheduler_->schedule([wp = weak_from_this()] { - // if (auto self = wp.lock()) { - // self->applyNextBlock(); - // } - // }); return true; } @@ -313,16 +299,7 @@ namespace kagome::network { if (handler) handler(Error::RESPONSE_WITHOUT_BLOCK_HEADER); return; } - // Check if body is provided - if (not block.header.has_value()) { - SL_ERROR(self->log_, - "Can't load blocks from {} starting from block {}: " - "Received block without body", - peer_id, - from); - if (handler) handler(Error::RESPONSE_WITHOUT_BLOCK_BODY); - return; - } + auto &header = block.header.value(); const auto &last_finalized_block = @@ -430,9 +407,7 @@ namespace kagome::network { self->ancestry_.emplace(header.parent_hash, block.hash); } - // handler(last_loaded_block); - - if (blocks.size() < 128) { + if (from.number + 20 >= last_loaded_block.number || blocks.size() < 127) { self->applyNextBlock(); handler(last_loaded_block); return; @@ -453,65 +428,114 @@ namespace kagome::network { void SynchronizerNewImpl::syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - const common::Buffer &key, + const std::vector &keys, SyncResultHandler &&handler) { if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { - if (not state_syncing_.load() || not key.empty()) { + if (not state_syncing_.load() + || (not keys.empty() && not keys[0].empty())) { state_syncing_.store(true); - if (key.empty()) { + if (keys[0].empty()) { sync_block_ = block; } - network::StateRequest request{block.hash, {key}, true}; + network::StateRequest request{block.hash, keys, true}; auto protocol = router_->getStateProtocol(); BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); SL_TRACE(log_, "State syncing started."); - auto response_handler = - [wp = weak_from_this(), - block, - peer_id, - handler = std::move(handler)](auto &&response_res) mutable { - auto self = wp.lock(); - if (not self) { - return; - } + auto response_handler = [wp = weak_from_this(), + block, + peer_id, + handler = std::move(handler)]( + auto &&response_res) mutable { + auto self = wp.lock(); + if (not self) { + return; + } - if (response_res.has_error()) { - SL_WARN(self->log_, - "State syncing failed with error: {}", - response_res.error().message()); - if (handler) handler(response_res.as_failure()); - return; - } + if (response_res.has_error()) { + SL_WARN(self->log_, + "State syncing failed with error: {}", + response_res.error().message()); + if (handler) handler(response_res.as_failure()); + return; + } - auto response = response_res.value().entries[0]; + for (unsigned i = 0; i < response_res.value().entries.size(); ++i) { + const auto &response = response_res.value().entries[i]; + + auto batch = + self->batches_store_.count(response.state_root) + ? std::get<2>(self->batches_store_[response.state_root]) + : self->storage_ + ->getPersistentBatchAt( + self->serializer_->getEmptyRootHash()) + .value(); + + if (response.entries.size()) { + SL_TRACE(self->log_, + "Syncing {}th item. Current key {}. Keys received {}.", + i, + response.entries[0].key.toHex(), + response.entries.size()); for (const auto &entry : response.entries) { - std::ignore = self->batch_->put(entry.key, entry.value); + std::ignore = batch->put(entry.key, entry.value); } - self->entries_ += response.entries.size(); - if (response.complete) { - auto res = self->batch_->commit(); - SL_INFO(self->log_, - "State syncing finished. Root hash: {}", - res.value().toHex()); - self->sync_method_ = - application::AppConfiguration::SyncMethod::Full; - if (handler) { - handler(block); - self->state_syncing_.store(false); - } + + if (!response.complete) { + self->batches_store_[response.state_root] = { + response.entries.back().key, i, batch}; } else { - SL_TRACE(self->log_, - "State syncing continues. {} entries loaded", - self->entries_); - self->syncState(peer_id, - block, - response.entries.back().key, - std::move(handler)); + self->batches_store_.erase(response.state_root); } - }; + } + + if (response.complete) { + auto res = batch->commit(); + SL_INFO( + self->log_, + "{} syncing finished. Root hash: {}. {}.", + i ? "Child state" : "State", + res.value().toHex(), + res.value() == response.state_root ? "Match" : "Don't match"); + if (res.value() != response.state_root) { + SL_INFO( + self->log_, "Should be {}", response.state_root.toHex()); + } + } + + if (!i) { + self->entries_ += response.entries.size(); + } + } + + std::map keymap; + for (const auto &[_, val] : self->batches_store_) { + unsigned i = std::get<1>(val); + keymap[i] = std::get<0>(val); + SL_TRACE(self->log_, "Index: {}, Key: {}", i, keymap[i]); + } + + std::vector keys; + for (const auto &[_, val] : keymap) { + keys.push_back(val); + } + + if (response_res.value().entries[0].complete) { + self->sync_method_ = + application::AppConfiguration::SyncMethod::Full; + if (handler) { + handler(block); + self->state_syncing_.store(false); + } + } else { + SL_TRACE(self->log_, + "State syncing continues. {} entries loaded", + self->entries_); + self->syncState(peer_id, block, keys, std::move(handler)); + } + }; protocol->request( peer_id, std::move(request), std::move(response_handler)); @@ -529,17 +553,6 @@ namespace kagome::network { return; } - bool false_val = false; - // if (not applying_in_progress_.compare_exchange_strong(false_val, true)) { - // SL_TRACE(log_, "Applying in progress"); - // return; - // } - // SL_TRACE(log_, "Begin applying"); - // auto cleanup = gsl::finally([this] { - // SL_TRACE(log_, "End applying"); - // applying_in_progress_ = false; - // }); - primitives::BlockHash hash; while (true) { @@ -614,12 +627,6 @@ namespace kagome::network { ancestry_.erase(hash); metric_import_queue_length_->set(known_blocks_.size()); - - // scheduler_->schedule([wp = weak_from_this()] { - // if (auto self = wp.lock()) { - // self->applyNextBlock(); - // } - // }); } size_t SynchronizerNewImpl::discardBlock( diff --git a/core/network/impl/synchronizer_new_impl.hpp b/core/network/impl/synchronizer_new_impl.hpp index 1a244768e6..fa0580fa81 100644 --- a/core/network/impl/synchronizer_new_impl.hpp +++ b/core/network/impl/synchronizer_new_impl.hpp @@ -95,7 +95,7 @@ namespace kagome::network { void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - const common::Buffer &key, + const std::vector &keys, SyncResultHandler &&handler) override; /// Loads blocks from peer {@param peer_id} since block {@param from} till @@ -169,7 +169,12 @@ namespace kagome::network { std::atomic_bool applying_in_progress_ = false; - std::shared_ptr batch_; + std::unordered_map< + storage::trie::RootHash, + std::tuple>> + batches_store_; size_t entries_{0}; }; diff --git a/core/network/synchronizer.hpp b/core/network/synchronizer.hpp index ef15c9fdea..bd2301d674 100644 --- a/core/network/synchronizer.hpp +++ b/core/network/synchronizer.hpp @@ -51,7 +51,7 @@ namespace kagome::network { virtual void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - const common::Buffer &key, + const std::vector &keys, SyncResultHandler &&handler) = 0; virtual void endSync() = 0; From 6c535b2204ccf6885d73e679e2b536ebc55235ce Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Fri, 10 Jun 2022 10:30:19 +0300 Subject: [PATCH 08/74] implement state response (#1164) --- core/network/impl/synchronizer_new_impl.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/network/impl/synchronizer_new_impl.cpp b/core/network/impl/synchronizer_new_impl.cpp index e8bdd573e7..498e5b4a95 100644 --- a/core/network/impl/synchronizer_new_impl.cpp +++ b/core/network/impl/synchronizer_new_impl.cpp @@ -431,6 +431,7 @@ namespace kagome::network { const std::vector &keys, SyncResultHandler &&handler) { if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { + // execute if not already started or iteration continues if (not state_syncing_.load() || (not keys.empty() && not keys[0].empty())) { state_syncing_.store(true); @@ -465,6 +466,7 @@ namespace kagome::network { for (unsigned i = 0; i < response_res.value().entries.size(); ++i) { const auto &response = response_res.value().entries[i]; + // get or create batch auto batch = self->batches_store_.count(response.state_root) ? std::get<2>(self->batches_store_[response.state_root]) @@ -473,6 +475,7 @@ namespace kagome::network { self->serializer_->getEmptyRootHash()) .value(); + // main storage entries size empty at child storage state syncing if (response.entries.size()) { SL_TRACE(self->log_, "Syncing {}th item. Current key {}. Keys received {}.", @@ -483,6 +486,7 @@ namespace kagome::network { std::ignore = batch->put(entry.key, entry.value); } + // store batch to continue at next response if (!response.complete) { self->batches_store_[response.state_root] = { response.entries.back().key, i, batch}; @@ -505,11 +509,13 @@ namespace kagome::network { } } + // just calculate state entries in main storage for trace log if (!i) { self->entries_ += response.entries.size(); } } + // not well formed way to place 0th batch key to front std::map keymap; for (const auto &[_, val] : self->batches_store_) { unsigned i = std::get<1>(val); From 6589f951c3622f7394c9a48414790d56d8e6f026 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Sat, 18 Jun 2022 20:42:49 +0300 Subject: [PATCH 09/74] fastsync working on polkadot (#1164) --- .../babe/impl/block_appender_impl.cpp | 31 ++++++++++++++++--- .../babe/impl/block_appender_impl.hpp | 2 ++ .../grandpa/impl/voting_round_impl.cpp | 2 ++ core/network/impl/peer_manager_impl.cpp | 4 +-- core/network/impl/synchronizer_new_impl.cpp | 17 +++++++--- core/network/impl/synchronizer_new_impl.hpp | 8 +++++ 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 74e93fa779..4d35aae96b 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -12,6 +12,7 @@ #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" #include "consensus/babe/types/slot.hpp" +#include "consensus/grandpa/impl/voting_round_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "primitives/common.hpp" #include "scale/scale.hpp" @@ -95,7 +96,8 @@ namespace kagome::consensus { OUTCOME_TRY(block_tree_->addExistingBlock(block_hash, header)); block_already_exists = true; - } else if (header_res.error() != blockchain::BlockTreeError::HEADER_NOT_FOUND) { + } else if (header_res.error() + != blockchain::BlockTreeError::HEADER_NOT_FOUND) { return header_res.as_failure(); } @@ -186,12 +188,33 @@ namespace kagome::consensus { SL_VERBOSE(logger_, "Justification received for block {}", primitives::BlockInfo(block.header.number, block_hash)); + if (not justifications_.empty()) { + std::vector to_remove; + for (const auto &[block_info, justification] : justifications_) { + auto res = grandpa_environment_->applyJustification(block_info, + justification); + if (res) { + to_remove.push_back(block_info); + } + } + if (not to_remove.empty()) { + for (const auto &item : to_remove) { + justifications_.erase(item); + } + } + } auto res = grandpa_environment_->applyJustification( primitives::BlockInfo(block.header.number, block_hash), b.justification.value()); if (res.has_error()) { - rollbackBlock(block_hash); - return res.as_failure(); + if (res.error() == grandpa::VotingRoundError::NOT_ENOUGH_WEIGHT) { + justifications_.emplace( + primitives::BlockInfo(block.header.number, block_hash), + b.justification.value()); + } else { + rollbackBlock(block_hash); + return res.as_failure(); + } } } @@ -217,4 +240,4 @@ namespace kagome::consensus { } } - } // namespace kagome::consensus +} // namespace kagome::consensus diff --git a/core/consensus/babe/impl/block_appender_impl.hpp b/core/consensus/babe/impl/block_appender_impl.hpp index f5bc78fce2..825e748ecf 100644 --- a/core/consensus/babe/impl/block_appender_impl.hpp +++ b/core/consensus/babe/impl/block_appender_impl.hpp @@ -53,6 +53,8 @@ namespace kagome::consensus { authority_update_observer_; std::shared_ptr babe_util_; + std::map justifications_; + log::Logger logger_; }; diff --git a/core/consensus/grandpa/impl/voting_round_impl.cpp b/core/consensus/grandpa/impl/voting_round_impl.cpp index 68a88cd50c..c4c0964914 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.cpp +++ b/core/consensus/grandpa/impl/voting_round_impl.cpp @@ -790,6 +790,7 @@ namespace kagome::consensus::grandpa { "vote={} target={}", vote.hash, signed_precommit.getBlockHash()); + SL_DEBUG(logger_, "first #{}", signed_precommit.getBlockHash()); } } else if (equivocators.emplace(signed_precommit.id).second) { @@ -804,6 +805,7 @@ namespace kagome::consensus::grandpa { "vote={} target={}", vote.hash.toHex(), signed_precommit.getBlockHash()); + SL_DEBUG(logger_, "second #{}", signed_precommit.getBlockHash()); } } else { diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index 47eb662414..24e121cdba 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -104,7 +104,7 @@ namespace kagome::network { "rating {}", peer_id, rating); - self->disconnectFromPeer(peer_id); + // self->disconnectFromPeer(peer_id); return; } self->processDiscoveredPeer(peer_id); @@ -377,7 +377,7 @@ namespace kagome::network { } if (peer_id != own_peer_info_.id) { peer_states_.erase(peer_id); - host_.disconnect(peer_id); + // host_.disconnect(peer_id); } } diff --git a/core/network/impl/synchronizer_new_impl.cpp b/core/network/impl/synchronizer_new_impl.cpp index 498e5b4a95..281325bd37 100644 --- a/core/network/impl/synchronizer_new_impl.cpp +++ b/core/network/impl/synchronizer_new_impl.cpp @@ -12,6 +12,7 @@ #include "network/helpers/peer_id_formatter.hpp" #include "network/types/block_attributes.hpp" #include "primitives/common.hpp" +#include "storage/changes_trie/changes_tracker.hpp" #include "storage/trie/serialization/trie_serializer.hpp" #include "storage/trie/trie_batches.hpp" #include "storage/trie/trie_storage.hpp" @@ -69,6 +70,7 @@ namespace kagome::network { const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr block_tree, + std::shared_ptr changes_tracker, std::shared_ptr block_appender, std::shared_ptr block_executor, std::shared_ptr serializer, @@ -77,6 +79,7 @@ namespace kagome::network { std::shared_ptr scheduler, std::shared_ptr hasher) : block_tree_(std::move(block_tree)), + trie_changes_tracker_(std::move(changes_tracker)), block_appender_(std::move(block_appender)), block_executor_(std::move(block_executor)), serializer_(std::move(serializer)), @@ -85,6 +88,7 @@ namespace kagome::network { scheduler_(std::move(scheduler)), hasher_(std::move(hasher)) { BOOST_ASSERT(block_tree_); + BOOST_ASSERT(trie_changes_tracker_ != nullptr); BOOST_ASSERT(block_executor_); BOOST_ASSERT(serializer_); BOOST_ASSERT(storage_); @@ -507,6 +511,7 @@ namespace kagome::network { SL_INFO( self->log_, "Should be {}", response.state_root.toHex()); } + self->trie_changes_tracker_->onBlockAdded(block.hash); } // just calculate state entries in main storage for trace log @@ -596,17 +601,21 @@ namespace kagome::network { if (sync_method_ == application::AppConfiguration::SyncMethod::Full && sync_block_ && block_info.number <= sync_block_.value().number) { + SL_WARN( + log_, "Skip {} till fast synchronized block", block_info.number); applyNextBlock(); } else { - if (sync_method_ == application::AppConfiguration::SyncMethod::Full - && sync_block_) { - sync_block_ = std::nullopt; - } auto applying_res = sync_method_ == application::AppConfiguration::SyncMethod::Full ? block_executor_->applyBlock(std::move(block)) : block_appender_->appendBlock(std::move(block)); + if (sync_method_ == application::AppConfiguration::SyncMethod::Full + && sync_block_ + && block_info.number == sync_block_.value().number + 1) { + sync_block_ = std::nullopt; + } + notifySubscribers({number, hash}, applying_res); if (not applying_res.has_value()) { diff --git a/core/network/impl/synchronizer_new_impl.hpp b/core/network/impl/synchronizer_new_impl.hpp index fa0580fa81..d351225d4d 100644 --- a/core/network/impl/synchronizer_new_impl.hpp +++ b/core/network/impl/synchronizer_new_impl.hpp @@ -18,6 +18,7 @@ #include "consensus/babe/block_executor.hpp" #include "metrics/metrics.hpp" #include "network/router.hpp" +#include "storage/changes_trie/changes_tracker.hpp" namespace kagome::application { class AppConfiguration; @@ -29,6 +30,10 @@ namespace kagome::storage::trie { class TrieStorage; } // namespace kagome::storage::trie +namespace kagome::storage::changes_trie { + class ChangesTracker; +} + namespace kagome::network { class SynchronizerNewImpl @@ -67,6 +72,7 @@ namespace kagome::network { const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr block_tree, + std::shared_ptr changes_tracker, std::shared_ptr block_appender, std::shared_ptr block_executor, std::shared_ptr serializer, @@ -127,6 +133,8 @@ namespace kagome::network { size_t discardBlock(const primitives::BlockHash &block); std::shared_ptr block_tree_; + std::shared_ptr + trie_changes_tracker_; std::shared_ptr block_appender_; std::shared_ptr block_executor_; std::shared_ptr serializer_; From 02142205b646925f622ee8de40022b3c565b28de Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 27 Jul 2022 09:53:13 +0300 Subject: [PATCH 10/74] clang-14 fixes --- core/consensus/authority/impl/authority_manager_impl.cpp | 5 +++-- core/consensus/babe/impl/babe_impl.cpp | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 2eb0f8431e..9f31d6817f 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -885,10 +885,11 @@ namespace kagome::authority { for (const auto &digest_item : header.digest) { std::ignore = visit_in_place( digest_item, - [&](const primitives::Consensus &consensus_message) + [&, num = header.number, hash = hash]( + const primitives::Consensus &consensus_message) -> outcome::result { ++count2; - return onConsensus(primitives::BlockInfo{header.number, hash}, + return onConsensus(primitives::BlockInfo{num, hash}, consensus_message); }, [](const auto &) { return outcome::success(); }); diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 85d794648a..ba1d6e4db2 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -334,13 +334,15 @@ namespace kagome::consensus::babe { outcome::result block_res) mutable { if (block_res.has_error()) { + SL_WARN(self->log_, + "Block result error: {}", + block_res.error().message()); return; } - const auto &block = block_res.value(); SL_INFO(self->log_, "Catching up is finished on block {}", - block); + block_res.value()); self->current_state_ = Babe::State::SYNCHRONIZED; self->was_synchronized_ = true; self->telemetry_->notifyWasSynchronized(); From 760fef3f6b50c8384e465d1441d70e5490977365 Mon Sep 17 00:00:00 2001 From: Alexander Krutikov Date: Wed, 27 Jul 2022 18:19:21 +0300 Subject: [PATCH 11/74] updates for PR --- .../authority/impl/authority_manager_impl.cpp | 8 +- .../grandpa/impl/environment_impl.cpp | 8 - .../grandpa/impl/environment_impl.hpp | 2 - .../grandpa/impl/voting_round_impl.cpp | 2 - core/injector/application_injector.cpp | 4 +- core/network/impl/CMakeLists.txt | 2 +- core/network/impl/peer_manager_impl.cpp | 4 +- core/network/impl/synchronizer_impl.cpp | 226 ++++++++++++------ core/network/impl/synchronizer_impl.hpp | 27 ++- core/network/types/grandpa_message.hpp | 21 -- core/utils/profiler.hpp | 4 +- test/core/network/synchronizer_test.cpp | 15 +- test/mock/core/host_api/host_api_mock.hpp | 1 - test/mock/core/network/synchronizer_mock.hpp | 8 +- 14 files changed, 200 insertions(+), 132 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 9f31d6817f..c23a14236d 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -699,10 +699,10 @@ namespace kagome::authority { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; }); } else { - // SL_WARN(log_, - // "Unknown consensus engine id in block {}: {}", - // block, - // message.consensus_engine_id.toString()); + SL_WARN(log_, + "Unknown consensus engine id in block {}: {}", + block, + message.consensus_engine_id.toString()); return outcome::success(); } } diff --git a/core/consensus/grandpa/impl/environment_impl.cpp b/core/consensus/grandpa/impl/environment_impl.cpp index d4370b2b43..2749c1f761 100644 --- a/core/consensus/grandpa/impl/environment_impl.cpp +++ b/core/consensus/grandpa/impl/environment_impl.cpp @@ -96,14 +96,6 @@ namespace kagome::consensus::grandpa { RoundNumber round_number) { network::CatchUpRequest message{.round_number = round_number, .voter_set_id = set_id}; - if (not recent_catchup_requests_.emplace(message.fingerprint()).second) { - SL_ERROR(logger_, - "Don't Send Duplicate Catch-Up-Request beginning with round {}", - round_number); - return outcome::success(); - } - SL_DEBUG( - logger_, "Send Catch-Up-Request beginning with round {}", round_number); transmitter_->sendCatchUpRequest(peer_id, std::move(message)); return outcome::success(); } diff --git a/core/consensus/grandpa/impl/environment_impl.hpp b/core/consensus/grandpa/impl/environment_impl.hpp index eeca0c8e98..1b50cd8bf9 100644 --- a/core/consensus/grandpa/impl/environment_impl.hpp +++ b/core/consensus/grandpa/impl/environment_impl.hpp @@ -105,8 +105,6 @@ namespace kagome::consensus::grandpa { std::shared_ptr transmitter_; std::weak_ptr justification_observer_; - std::set recent_catchup_requests_; - log::Logger logger_; }; diff --git a/core/consensus/grandpa/impl/voting_round_impl.cpp b/core/consensus/grandpa/impl/voting_round_impl.cpp index c4c0964914..68a88cd50c 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.cpp +++ b/core/consensus/grandpa/impl/voting_round_impl.cpp @@ -790,7 +790,6 @@ namespace kagome::consensus::grandpa { "vote={} target={}", vote.hash, signed_precommit.getBlockHash()); - SL_DEBUG(logger_, "first #{}", signed_precommit.getBlockHash()); } } else if (equivocators.emplace(signed_precommit.id).second) { @@ -805,7 +804,6 @@ namespace kagome::consensus::grandpa { "vote={} target={}", vote.hash.toHex(), signed_precommit.getBlockHash()); - SL_DEBUG(logger_, "second #{}", signed_precommit.getBlockHash()); } } else { diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 14764d841c..68d5ba4584 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -94,7 +94,7 @@ #include "network/impl/router_libp2p.hpp" #include "network/impl/state_protocol_observer_impl.hpp" #include "network/impl/sync_protocol_observer_impl.hpp" -#include "network/impl/synchronizer_new_impl.hpp" +#include "network/impl/synchronizer_impl.hpp" #include "network/impl/transactions_transmitter_impl.hpp" #include "network/sync_protocol_observer.hpp" #include "offchain/impl/offchain_local_storage.hpp" @@ -1239,7 +1239,7 @@ namespace { static auto last_finalized_hash = get_last_finalized_hash(injector); return get_babe_configuration(last_finalized_hash, babe_api); }), - di::bind.template to(), + di::bind.template to(), di::bind.template to(), di::bind.template to(), di::bind.template to(), diff --git a/core/network/impl/CMakeLists.txt b/core/network/impl/CMakeLists.txt index bd120be17f..9d69849cfe 100644 --- a/core/network/impl/CMakeLists.txt +++ b/core/network/impl/CMakeLists.txt @@ -6,7 +6,7 @@ add_subdirectory(protocols) add_library(synchronizer - synchronizer_new_impl.cpp + synchronizer_impl.cpp ) target_link_libraries(synchronizer logger diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index 24e121cdba..47eb662414 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -104,7 +104,7 @@ namespace kagome::network { "rating {}", peer_id, rating); - // self->disconnectFromPeer(peer_id); + self->disconnectFromPeer(peer_id); return; } self->processDiscoveredPeer(peer_id); @@ -377,7 +377,7 @@ namespace kagome::network { } if (peer_id != own_peer_info_.id) { peer_states_.erase(peer_id); - // host_.disconnect(peer_id); + host_.disconnect(peer_id); } } diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 2adaed864f..e57789bcfb 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -12,6 +12,7 @@ #include "network/helpers/peer_id_formatter.hpp" #include "network/types/block_attributes.hpp" #include "primitives/common.hpp" +#include "storage/changes_trie/changes_tracker.hpp" #include "storage/trie/serialization/trie_serializer.hpp" #include "storage/trie/trie_batches.hpp" #include "storage/trie/trie_storage.hpp" @@ -69,6 +70,7 @@ namespace kagome::network { const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr block_tree, + std::shared_ptr changes_tracker, std::shared_ptr block_appender, std::shared_ptr block_executor, std::shared_ptr serializer, @@ -77,6 +79,7 @@ namespace kagome::network { std::shared_ptr scheduler, std::shared_ptr hasher) : block_tree_(std::move(block_tree)), + trie_changes_tracker_(std::move(changes_tracker)), block_appender_(std::move(block_appender)), block_executor_(std::move(block_executor)), serializer_(std::move(serializer)), @@ -85,6 +88,7 @@ namespace kagome::network { scheduler_(std::move(scheduler)), hasher_(std::move(hasher)) { BOOST_ASSERT(block_tree_); + BOOST_ASSERT(trie_changes_tracker_); BOOST_ASSERT(block_executor_); BOOST_ASSERT(serializer_); BOOST_ASSERT(storage_); @@ -96,9 +100,6 @@ namespace kagome::network { sync_method_ = app_config.syncMethod(); - batch_ = - storage_->getPersistentBatchAt(serializer_->getEmptyRootHash()).value(); - // Register metrics metrics_registry_->registerGaugeFamily( kImportQueueLength, "Number of blocks submitted to the import queue"); @@ -166,6 +167,11 @@ namespace kagome::network { const libp2p::peer::PeerId &peer_id, Synchronizer::SyncResultHandler &&handler, bool subscribe_to_block) { + if (not state_syncing_.load() && not syncing_.load()) { + syncing_.store(true); + } else { + return false; + } // Subscribe on demand if (subscribe_to_block) { subscribeToBlock(block_info, std::move(handler)); @@ -274,6 +280,11 @@ namespace kagome::network { const primitives::BlockHeader &header, const libp2p::peer::PeerId &peer_id, Synchronizer::SyncResultHandler &&handler) { + if (not state_syncing_.load() && not syncing_.load()) { + syncing_.store(true); + } else { + return false; + } auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); const primitives::BlockInfo block_info(header.number, block_hash); @@ -878,62 +889,121 @@ namespace kagome::network { void SynchronizerImpl::syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - const common::Buffer &key, + const std::vector &keys, SyncResultHandler &&handler) { if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { - if (not state_syncing_.load() || not key.empty()) { + // execute if not already started or iteration continues + if (not state_syncing_.load() + || (not keys.empty() && not keys[0].empty())) { state_syncing_.store(true); - network::StateRequest request{block.hash, {key}, true}; + if (keys[0].empty()) { + sync_block_ = block; + } + network::StateRequest request{block.hash, keys, true}; auto protocol = router_->getStateProtocol(); BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); SL_TRACE(log_, "State syncing started."); - auto response_handler = - [wp = weak_from_this(), - block, - peer_id, - handler = std::move(handler)](auto &&response_res) mutable { - auto self = wp.lock(); - if (not self) { - return; - } + auto response_handler = [wp = weak_from_this(), + block, + peer_id, + handler = std::move(handler)]( + auto &&response_res) mutable { + auto self = wp.lock(); + if (not self) { + return; + } - if (response_res.has_error()) { - SL_WARN(self->log_, - "State syncing failed with error: {}", - response_res.error().message()); - if (handler) handler(response_res.as_failure()); - return; - } + if (response_res.has_error()) { + SL_WARN(self->log_, + "State syncing failed with error: {}", + response_res.error().message()); + if (handler) handler(response_res.as_failure()); + return; + } - auto response = response_res.value().entries[0]; + for (unsigned i = 0; i < response_res.value().entries.size(); ++i) { + const auto &response = response_res.value().entries[i]; + + // get or create batch + auto batch = + self->batches_store_.count(response.state_root) + ? std::get<2>(self->batches_store_[response.state_root]) + : self->storage_ + ->getPersistentBatchAt( + self->serializer_->getEmptyRootHash()) + .value(); + + // main storage entries size empty at child storage state syncing + if (response.entries.size()) { + SL_TRACE(self->log_, + "Syncing {}th item. Current key {}. Keys received {}.", + i, + response.entries[0].key.toHex(), + response.entries.size()); for (const auto &entry : response.entries) { - std::ignore = self->batch_->put(entry.key, entry.value); + std::ignore = batch->put(entry.key, entry.value); } - self->entries_ += response.entries.size(); - if (response.complete) { - auto res = self->batch_->commit(); - SL_INFO(self->log_, - "State syncing finished. Root hash: {}", - res.value().toHex()); - self->sync_method_ = - application::AppConfiguration::SyncMethod::Full; - if (handler) { - handler(block); - self->state_syncing_.store(false); - } + + // store batch to continue at next response + if (!response.complete) { + self->batches_store_[response.state_root] = { + response.entries.back().key, i, batch}; } else { - SL_TRACE(self->log_, - "State syncing continues. {} entries loaded", - self->entries_); - self->syncState(peer_id, - block, - response.entries.back().key, - std::move(handler)); + self->batches_store_.erase(response.state_root); } - }; + } + + if (response.complete) { + auto res = batch->commit(); + SL_INFO( + self->log_, + "{} syncing finished. Root hash: {}. {}.", + i ? "Child state" : "State", + res.value().toHex(), + res.value() == response.state_root ? "Match" : "Don't match"); + if (res.value() != response.state_root) { + SL_INFO( + self->log_, "Should be {}", response.state_root.toHex()); + } + self->trie_changes_tracker_->onBlockAdded(block.hash); + } + + // just calculate state entries in main storage for trace log + if (!i) { + self->entries_ += response.entries.size(); + } + } + + // not well formed way to place 0th batch key to front + std::map keymap; + for (const auto &[_, val] : self->batches_store_) { + unsigned i = std::get<1>(val); + keymap[i] = std::get<0>(val); + SL_TRACE(self->log_, "Index: {}, Key: {}", i, keymap[i]); + } + + std::vector keys; + for (const auto &[_, val] : keymap) { + keys.push_back(val); + } + + if (response_res.value().entries[0].complete) { + self->sync_method_ = + application::AppConfiguration::SyncMethod::Full; + if (handler) { + handler(block); + self->state_syncing_.store(false); + } + } else { + SL_TRACE(self->log_, + "State syncing continues. {} entries loaded", + self->entries_); + self->syncState(peer_id, block, keys, std::move(handler)); + } + }; protocol->request( peer_id, std::move(request), std::move(response_handler)); @@ -1006,33 +1076,48 @@ namespace kagome::network { } else { const BlockInfo block_info(block.header->number, block.hash); - auto applying_res = - sync_method_ == application::AppConfiguration::SyncMethod::Full - ? block_executor_->applyBlock(std::move(block)) - : block_appender_->appendBlock(std::move(block)); - - notifySubscribers({number, hash}, applying_res); - - if (not applying_res.has_value()) { - if (applying_res - != outcome::failure(blockchain::BlockTreeError::BLOCK_EXISTS)) { - notifySubscribers({number, hash}, applying_res.as_failure()); - auto n = discardBlock(block_info.hash); - SL_WARN( - log_, - "Block {} {} been discarded: {}", - block_info, - n ? fmt::format("and {} others have", n) : fmt::format("has"), - applying_res.error().message()); - if (handler) handler(Error::DISCARDED_BLOCK); + if (sync_method_ == application::AppConfiguration::SyncMethod::Full + && sync_block_ && block_info.number <= sync_block_.value().number) { + SL_WARN( + log_, "Skip {} till fast synchronized block", block_info.number); + applyNextBlock(); + } else { + auto applying_res = + sync_method_ == application::AppConfiguration::SyncMethod::Full + ? block_executor_->applyBlock(std::move(block)) + : block_appender_->appendBlock(std::move(block)); + + if (sync_method_ == application::AppConfiguration::SyncMethod::Full + && sync_block_ + && block_info.number == sync_block_.value().number + 1) { + sync_block_ = std::nullopt; + } + + notifySubscribers({number, hash}, applying_res); + + if (not applying_res.has_value()) { + if (applying_res + != outcome::failure(blockchain::BlockTreeError::BLOCK_EXISTS)) { + notifySubscribers({number, hash}, applying_res.as_failure()); + auto n = discardBlock(block_info.hash); + SL_WARN( + log_, + "Block {} {} been discarded: {}", + block_info, + n ? fmt::format("and {} others have", n) : fmt::format("has"), + applying_res.error().message()); + if (handler) handler(Error::DISCARDED_BLOCK); + } else { + SL_DEBUG(log_, "Block {} is skipped as existing", block_info); + if (handler) handler(block_info); + // applyNextBlock(); + } } else { - SL_DEBUG(log_, "Block {} is skipped as existing", block_info); + telemetry_->notifyBlockImported( + block_info, telemetry::BlockOrigin::kNetworkInitialSync); if (handler) handler(block_info); + // applyNextBlock(); } - } else { - telemetry_->notifyBlockImported( - block_info, telemetry::BlockOrigin::kNetworkInitialSync); - if (handler) handler(block_info); } } } @@ -1047,7 +1132,6 @@ namespace kagome::network { SL_TRACE(log_, "{} blocks in queue", known_blocks_.size()); } metric_import_queue_length_->set(known_blocks_.size()); - scheduler_->schedule([wp = weak_from_this()] { if (auto self = wp.lock()) { self->applyNextBlock(); @@ -1117,7 +1201,6 @@ namespace kagome::network { } metric_import_queue_length_->set(known_blocks_.size()); - return affected; } @@ -1138,7 +1221,8 @@ namespace kagome::network { } } - // Remove blocks whose numbers equal finalized one, excluding finalized one + // Remove blocks whose numbers equal finalized one, excluding finalized + // one auto range = generations_.equal_range(finalized_block.number); for (auto it = range.first; it != range.second;) { auto cit = it++; diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 430f1306c5..6cd0742259 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -25,6 +25,10 @@ namespace kagome::application { class AppConfiguration; } +namespace kagome::storage::changes_trie { + class ChangesTracker; +} + namespace kagome::storage::trie { class PersistentTrieBatch; class TrieSerializer; @@ -69,6 +73,7 @@ namespace kagome::network { const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr block_tree, + std::shared_ptr changes_tracker, std::shared_ptr block_appender, std::shared_ptr block_executor, std::shared_ptr serializer, @@ -101,9 +106,12 @@ namespace kagome::network { std::optional limit, SyncResultHandler &&handler) override; + /// Enqueues loading and applying state key values starting from {@keys} for + /// block {@param block} from peer {@param peer_id}. + /// If finished, {@param handler} be called void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block, - const common::Buffer &key, + const std::vector &keys, SyncResultHandler &&handler) override; /// Finds best common block with peer {@param peer_id} in provided interval. @@ -136,6 +144,10 @@ namespace kagome::network { std::optional limit, SyncResultHandler &&handler); + void endSync() override { + syncing_.store(false); + } + private: /// Subscribes handler for block with provided {@param block_info} /// {@param handler} will be called When block is received or discarded @@ -170,8 +182,9 @@ namespace kagome::network { const libp2p::peer::PeerId &peer_id, const BlocksRequest::Fingerprint &fingerprint); - const application::AppConfiguration &app_config_; std::shared_ptr block_tree_; + std::shared_ptr + trie_changes_tracker_; std::shared_ptr block_appender_; std::shared_ptr block_executor_; std::shared_ptr serializer_; @@ -188,6 +201,7 @@ namespace kagome::network { log::Logger log_ = log::createLogger("Synchronizer", "synchronizer"); telemetry::Telemetry telemetry_ = telemetry::createTelemetryService(); + std::atomic_bool syncing_ = false; std::atomic_bool state_syncing_ = false; bool node_is_shutting_down_ = false; @@ -201,6 +215,8 @@ namespace kagome::network { // Already known (enqueued) but is not applied yet std::unordered_map known_blocks_; + std::optional sync_block_; + // Blocks grouped by number std::multimap generations_; @@ -230,7 +246,12 @@ namespace kagome::network { std::set> recent_requests_; - std::shared_ptr batch_; + std::unordered_map< + storage::trie::RootHash, + std::tuple>> + batches_store_; size_t entries_{0}; }; diff --git a/core/network/types/grandpa_message.hpp b/core/network/types/grandpa_message.hpp index a4d9d32509..4a3cf787fe 100644 --- a/core/network/types/grandpa_message.hpp +++ b/core/network/types/grandpa_message.hpp @@ -138,25 +138,4 @@ namespace kagome::network { } // namespace kagome::network -template <> -struct std::hash { - auto operator()( - const kagome::network::CatchUpRequest &catch_up_request) const { - auto result = std::hash()( - catch_up_request.round_number); - - boost::hash_combine( - result, - std::hash()( - catch_up_request.voter_set_id)); - - return result; - } -}; - -inline kagome::network::CatchUpRequest::Fingerprint -kagome::network::CatchUpRequest::fingerprint() const { - return std::hash()(*this); -} - #endif // KAGOME_NETWORK_GRANDPAMESSAGE diff --git a/core/utils/profiler.hpp b/core/utils/profiler.hpp index 6f3ba1b62a..a0b76b055c 100644 --- a/core/utils/profiler.hpp +++ b/core/utils/profiler.hpp @@ -23,9 +23,9 @@ class TicToc { str += "at line " + std::to_string(line); } log_->info( - "{} lasted for {} microsec", + "{} lasted for {} sec", str, - std::chrono::duration_cast(t_ - prev).count()); + std::chrono::duration_cast(t_ - prev).count()); } ~TicToc() { diff --git a/test/core/network/synchronizer_test.cpp b/test/core/network/synchronizer_test.cpp index e550c9c4a2..d25e79d9f5 100644 --- a/test/core/network/synchronizer_test.cpp +++ b/test/core/network/synchronizer_test.cpp @@ -17,11 +17,13 @@ #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/network/protocols/sync_protocol_mock.hpp" #include "mock/core/network/router_mock.hpp" +#include "mock/core/storage/changes_trie/changes_tracker_mock.hpp" #include "mock/core/storage/trie/serialization/trie_serializer_mock.hpp" #include "mock/core/storage/trie/trie_batches_mock.hpp" #include "mock/core/storage/trie/trie_storage_mock.hpp" #include "network/impl/synchronizer_impl.hpp" #include "primitives/common.hpp" +#include "storage/changes_trie/changes_tracker.hpp" #include "storage/trie/serialization/polkadot_codec.hpp" #include "testutil/literals.hpp" #include "testutil/prepare_loggers.hpp" @@ -73,20 +75,11 @@ class SynchronizerTest EXPECT_CALL(app_config, syncMethod()) .WillOnce(Return(application::AppConfiguration::SyncMethod::Full)); - EXPECT_CALL(*serializer, getEmptyRootHash()) - .Times(2) - .WillRepeatedly( - Return(trie::PolkadotCodec().hash256(common::Buffer{0}))); - - outcome::result> batch = - std::make_unique(); - EXPECT_CALL(*storage, getPersistentBatchAt(serializer->getEmptyRootHash())) - .WillOnce(Return(testing::ByMove(std::move(batch)))); - synchronizer = std::make_shared(app_config, app_state_manager, block_tree, + changes_tracker, block_appender, block_executor, serializer, @@ -101,6 +94,8 @@ class SynchronizerTest std::make_shared(); std::shared_ptr block_tree = std::make_shared(); + std::shared_ptr changes_tracker = + std::make_shared(); std::shared_ptr block_appender = std::make_shared(); std::shared_ptr block_executor = diff --git a/test/mock/core/host_api/host_api_mock.hpp b/test/mock/core/host_api/host_api_mock.hpp index ea215c889c..722e4eb75f 100644 --- a/test/mock/core/host_api/host_api_mock.hpp +++ b/test/mock/core/host_api/host_api_mock.hpp @@ -7,7 +7,6 @@ #define KAGOME_TEST_CORE_RUNTIME_MOCK_HOST_API_HPP_ #include "host_api/host_api.hpp" -#include "runtime/types.hpp" #include diff --git a/test/mock/core/network/synchronizer_mock.hpp b/test/mock/core/network/synchronizer_mock.hpp index bab750289c..81471fbedb 100644 --- a/test/mock/core/network/synchronizer_mock.hpp +++ b/test/mock/core/network/synchronizer_mock.hpp @@ -58,16 +58,18 @@ namespace kagome::network { syncState, (const libp2p::peer::PeerId &, const primitives::BlockInfo &, - const common::Buffer &, + const std::vector &, const SyncResultHandler &), ()); void syncState(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &block_info, - const common::Buffer &key, + const std::vector &keys, SyncResultHandler &&handler) override { - return syncState(peer_id, block_info, key, handler); + return syncState(peer_id, block_info, keys, handler); } + + MOCK_METHOD(void, endSync, ()); }; } // namespace kagome::network From bff1ee2a5879f2be957029ad4614e6b8809cdf33 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Thu, 4 Aug 2022 19:19:21 +0300 Subject: [PATCH 12/74] fix: processing of epoch digests during fast syncing Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/block_appender.hpp | 4 + .../babe/impl/block_appender_impl.cpp | 138 ++++++++++++------ .../babe/impl/block_appender_impl.hpp | 14 +- .../babe/impl/block_executor_impl.cpp | 1 - core/network/impl/stream_engine.hpp | 2 - 5 files changed, 106 insertions(+), 53 deletions(-) diff --git a/core/consensus/babe/block_appender.hpp b/core/consensus/babe/block_appender.hpp index 9a4b57c11f..dcaad5ee11 100644 --- a/core/consensus/babe/block_appender.hpp +++ b/core/consensus/babe/block_appender.hpp @@ -16,6 +16,10 @@ namespace kagome::consensus { virtual ~BlockAppender() = default; virtual outcome::result appendBlock(primitives::BlockData &&b) = 0; + + virtual outcome::result applyJustification( + const primitives::BlockInfo &block_info, + const primitives::Justification &justification) = 0; }; } // namespace kagome::consensus diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 4d35aae96b..c62e0ad39c 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -9,6 +9,7 @@ #include "blockchain/block_tree_error.hpp" #include "blockchain/impl/common.hpp" +#include "consensus/babe/consistency_keeper.hpp" #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" #include "consensus/babe/types/slot.hpp" @@ -29,11 +30,6 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus, BlockAppenderImpl::Error, e) { return "Unknown error"; } -namespace { - constexpr const char *kBlockExecutionTime = - "kagome_block_verification_and_import_time"; -} - namespace kagome::consensus { BlockAppenderImpl::BlockAppenderImpl( @@ -44,7 +40,8 @@ namespace kagome::consensus { std::shared_ptr hasher, std::shared_ptr authority_update_observer, - std::shared_ptr babe_util) + std::shared_ptr babe_util, + std::shared_ptr consistency_keeper) : block_tree_{std::move(block_tree)}, babe_configuration_{std::move(configuration)}, block_validator_{std::move(block_validator)}, @@ -52,6 +49,7 @@ namespace kagome::consensus { hasher_{std::move(hasher)}, authority_update_observer_{std::move(authority_update_observer)}, babe_util_(std::move(babe_util)), + consistency_keeper_(std::move(consistency_keeper)), logger_{log::createLogger("BlockAppender", "block_appender")} { BOOST_ASSERT(block_tree_ != nullptr); BOOST_ASSERT(babe_configuration_ != nullptr); @@ -60,6 +58,7 @@ namespace kagome::consensus { BOOST_ASSERT(hasher_ != nullptr); BOOST_ASSERT(authority_update_observer_ != nullptr); BOOST_ASSERT(babe_util_ != nullptr); + BOOST_ASSERT(consistency_keeper_ != nullptr); BOOST_ASSERT(logger_ != nullptr); } @@ -71,10 +70,14 @@ namespace kagome::consensus { } auto &header = b.header.value(); + auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); + + primitives::BlockInfo block_info(header.number, block_hash); + if (auto header_res = block_tree_->getBlockHeader(header.parent_hash); header_res.has_error() && header_res.error() == blockchain::BlockTreeError::HEADER_NOT_FOUND) { - logger_->warn("Skipping a block with unknown parent"); + logger_->warn("Skipping a block {} with unknown parent", block_info); return Error::PARENT_NOT_FOUND; } else if (header_res.has_error()) { return header_res.as_failure(); @@ -83,16 +86,12 @@ namespace kagome::consensus { // get current time to measure performance if block execution auto t_start = std::chrono::high_resolution_clock::now(); - auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); - bool block_already_exists = false; // check if block header already exists. If so, do not append if (auto header_res = block_tree_->getBlockHeader(block_hash); header_res.has_value()) { - SL_DEBUG(logger_, - "Skip existing block: {}", - primitives::BlockInfo(header.number, block_hash)); + SL_DEBUG(logger_, "Skip existing header of block: {}", block_info); OUTCOME_TRY(block_tree_->addExistingBlock(block_hash, header)); block_already_exists = true; @@ -101,6 +100,8 @@ namespace kagome::consensus { return header_res.as_failure(); } + auto consistency_guard = consistency_keeper_->start(block_info); + primitives::Block block{.header = std::move(header)}; OUTCOME_TRY(babe_digests, getBabeDigests(block.header)); @@ -108,11 +109,46 @@ namespace kagome::consensus { const auto &babe_header = babe_digests.second; auto slot_number = babe_header.slot_number; + + babe_util_->syncEpoch([&] { + auto res = block_tree_->getBlockHeader(primitives::BlockNumber(1)); + if (res.has_error()) { + if (block.header.number == 1) { + SL_TRACE(logger_, + "First block slot is {}: it is first block (at executing)", + slot_number); + return std::tuple(slot_number, false); + } else { + SL_TRACE(logger_, + "First block slot is {}: no first block (at executing)", + babe_util_->getCurrentSlot()); + return std::tuple(babe_util_->getCurrentSlot(), false); + } + } + + const auto &first_block_header = res.value(); + auto babe_digest_res = consensus::getBabeDigests(first_block_header); + BOOST_ASSERT_MSG(babe_digest_res.has_value(), + "Any non genesis block must contain babe digest"); + auto first_slot_number = babe_digest_res.value().second.slot_number; + + auto is_first_block_finalized = + block_tree_->getLastFinalized().number > 0; + + SL_TRACE( + logger_, + "First block slot is {}: by {}finalized first block (at executing)", + first_slot_number, + is_first_block_finalized ? "" : "non-"); + return std::tuple(first_slot_number, is_first_block_finalized); + }); + auto epoch_number = babe_util_->slotToEpoch(slot_number); - logger_->info( + SL_DEBUG( + logger_, "Applying block {} ({} in slot {}, epoch {})", // - primitives::BlockInfo(block.header.number, block_hash), + block_info, babe_header.slotType() == SlotType::Primary ? "primary" : babe_header.slotType() == SlotType::SecondaryVRF ? "secondary-vrf" : babe_header.slotType() == SlotType::SecondaryPlain ? "secondary-plain" @@ -127,7 +163,7 @@ namespace kagome::consensus { SL_TRACE(logger_, "Actual epoch digest to apply block {} (slot {}, epoch {}). " "Randomness: {}", - primitives::BlockInfo(block.header.number, block_hash), + block_info, slot_number, epoch_number, this_block_epoch_descriptor.randomness); @@ -148,8 +184,8 @@ namespace kagome::consensus { SL_VERBOSE(logger_, "Next epoch digest has gotten in block {} (slot {}). " "Randomness: {}", + block_info, slot_number, - primitives::BlockInfo(block.header.number, block_hash), next_epoch_digest.randomness); } @@ -160,6 +196,8 @@ namespace kagome::consensus { auto parent = block_tree_->getBlockHeader(block.header.parent_hash).value(); + auto last_finalized_block = block_tree_->getLastFinalized(); + if (not block_already_exists) { OUTCOME_TRY(block_tree_->addBlockHeader(block.header)); } @@ -177,7 +215,10 @@ namespace kagome::consensus { }, [](const auto &) { return outcome::success(); }); if (res.has_error()) { - rollbackBlock(block_hash); + SL_ERROR(logger_, + "Error while processing consensus digests of block {}: {}", + block_hash, + res.error().message()); return res.as_failure(); } } @@ -185,16 +226,16 @@ namespace kagome::consensus { // apply justification if any (must be done strictly after block will be // added and his consensus-digests will be handled) if (b.justification.has_value()) { - SL_VERBOSE(logger_, - "Justification received for block {}", - primitives::BlockInfo(block.header.number, block_hash)); + SL_VERBOSE(logger_, "Justification received for block {}", block_info); + + // try to apply left in justification store values first if (not justifications_.empty()) { std::vector to_remove; - for (const auto &[block_info, justification] : justifications_) { - auto res = grandpa_environment_->applyJustification(block_info, - justification); + for (const auto &[block_justified_for, justification] : + justifications_) { + auto res = applyJustification(block_justified_for, justification); if (res) { - to_remove.push_back(block_info); + to_remove.push_back(block_justified_for); } } if (not to_remove.empty()) { @@ -203,41 +244,44 @@ namespace kagome::consensus { } } } - auto res = grandpa_environment_->applyJustification( - primitives::BlockInfo(block.header.number, block_hash), - b.justification.value()); + + auto res = applyJustification(block_info, b.justification.value()); + if (res.has_error()) { - if (res.error() == grandpa::VotingRoundError::NOT_ENOUGH_WEIGHT) { - justifications_.emplace( - primitives::BlockInfo(block.header.number, block_hash), - b.justification.value()); + if (res + == outcome::failure(grandpa::VotingRoundError::NOT_ENOUGH_WEIGHT)) { + justifications_.emplace(block_info, b.justification.value()); } else { - rollbackBlock(block_hash); return res.as_failure(); } + } else { + // safely could be remove if current justification applied successfully + justifications_.clear(); } } - auto t_end = std::chrono::high_resolution_clock::now(); + SL_DEBUG(logger_, + "Imported header of block {} within {} ms", + block_info, + std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - t_start) + .count()); + + if (block_info.number % 512 == 0) { + SL_INFO(logger_, + "Imported another 512 headers of blocks. Last one imported of {}", + block_info); + } - logger_->info( - "Imported block {} within {} ms", - primitives::BlockInfo(block.header.number, block_hash), - std::chrono::duration_cast(t_end - t_start) - .count()); + consistency_guard.commit(); return outcome::success(); } - void BlockAppenderImpl::rollbackBlock( - const primitives::BlockHash &block_hash) { - auto removal_res = block_tree_->removeLeaf(block_hash); - if (removal_res.has_error()) { - SL_WARN(logger_, - "Rolling back of block {} is failed: {}", - block_hash, - removal_res.error().message()); - } + outcome::result BlockAppenderImpl::applyJustification( + const primitives::BlockInfo &block_info, + const primitives::Justification &justification) { + return grandpa_environment_->applyJustification(block_info, justification); } } // namespace kagome::consensus diff --git a/core/consensus/babe/impl/block_appender_impl.hpp b/core/consensus/babe/impl/block_appender_impl.hpp index 825e748ecf..9f331352f1 100644 --- a/core/consensus/babe/impl/block_appender_impl.hpp +++ b/core/consensus/babe/impl/block_appender_impl.hpp @@ -22,6 +22,9 @@ #include "primitives/block_header.hpp" namespace kagome::consensus { + namespace babe { + class ConsistencyKeeper; + } class BlockAppenderImpl : public BlockAppender, @@ -37,13 +40,16 @@ namespace kagome::consensus { std::shared_ptr hasher, std::shared_ptr authority_update_observer, - std::shared_ptr babe_util); + std::shared_ptr babe_util, + std::shared_ptr consistency_keeper); outcome::result appendBlock(primitives::BlockData &&b) override; - private: - void rollbackBlock(const primitives::BlockHash &block_hash); + outcome::result applyJustification( + const primitives::BlockInfo &block_info, + const primitives::Justification &justification) override; + private: std::shared_ptr block_tree_; std::shared_ptr babe_configuration_; std::shared_ptr block_validator_; @@ -52,7 +58,9 @@ namespace kagome::consensus { std::shared_ptr authority_update_observer_; std::shared_ptr babe_util_; + std::shared_ptr consistency_keeper_; + // Justification Store for Future Applying std::map justifications_; log::Logger logger_; diff --git a/core/consensus/babe/impl/block_executor_impl.cpp b/core/consensus/babe/impl/block_executor_impl.cpp index 6737653b08..a5506fac03 100644 --- a/core/consensus/babe/impl/block_executor_impl.cpp +++ b/core/consensus/babe/impl/block_executor_impl.cpp @@ -8,7 +8,6 @@ #include #include "blockchain/block_tree_error.hpp" -#include "blockchain/impl/common.hpp" #include "consensus/babe/consistency_keeper.hpp" #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" diff --git a/core/network/impl/stream_engine.hpp b/core/network/impl/stream_engine.hpp index 4adf98f6b7..1123c0b1fc 100644 --- a/core/network/impl/stream_engine.hpp +++ b/core/network/impl/stream_engine.hpp @@ -488,8 +488,6 @@ namespace kagome::network { if (stream_res == outcome::failure( std::make_error_code(std::errc::not_connected))) { - self->logger_->error("GOTCHA!"); // FIXME - self->del(peer_id); return; } From 1e3db6db9173260edd15c9972062cc852cba6682 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 5 Aug 2022 16:13:01 +0300 Subject: [PATCH 13/74] fix: tests Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/impl/block_appender_impl.cpp | 4 ---- test/mock/core/consensus/babe/block_appender_mock.hpp | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index c62e0ad39c..821b713819 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -194,10 +194,6 @@ namespace kagome::consensus { // block should be applied without last digest which contains the seal block_without_seal_digest.header.digest.pop_back(); - auto parent = block_tree_->getBlockHeader(block.header.parent_hash).value(); - - auto last_finalized_block = block_tree_->getLastFinalized(); - if (not block_already_exists) { OUTCOME_TRY(block_tree_->addBlockHeader(block.header)); } diff --git a/test/mock/core/consensus/babe/block_appender_mock.hpp b/test/mock/core/consensus/babe/block_appender_mock.hpp index a519ef45b3..e0d740b140 100644 --- a/test/mock/core/consensus/babe/block_appender_mock.hpp +++ b/test/mock/core/consensus/babe/block_appender_mock.hpp @@ -21,6 +21,12 @@ namespace kagome::consensus { outcome::result appendBlock(primitives::BlockData &&block) override { return appendBlock(block); } + + MOCK_METHOD(outcome::result, + applyJustification, + (const primitives::BlockInfo &block_info, + const primitives::Justification &justification), + (override)); }; } // namespace kagome::consensus From 91e2f4fac03a888e4e4b5f9e06706fa8fee1d9e6 Mon Sep 17 00:00:00 2001 From: Alexander Lednev <57529355+iceseer@users.noreply.github.com> Date: Mon, 8 Aug 2022 21:42:44 +0300 Subject: [PATCH 14/74] Build fix for external blob (#1306) Signed-off-by: iceseer --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f781c8f5f9..146a060746 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,6 +155,7 @@ kagome_install_setup( core/scale core/storage core/subscription + core/utils ) include(CMakePackageConfigHelpers) From bf1d6187711621bf8877499a5e8a73ac7d1bd769 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov Date: Tue, 9 Aug 2022 12:11:01 +0300 Subject: [PATCH 15/74] Feature: change logging level over RPC in real time #1305 * feature: change logging level over RPC Signed-off-by: Dmitriy Khaustov aka xDimon --- core/api/service/CMakeLists.txt | 1 + core/api/service/chain/chain_api.hpp | 2 +- core/api/service/internal/CMakeLists.txt | 13 ++++ .../internal/impl/internal_api_impl.cpp | 23 ++++++ .../internal/impl/internal_api_impl.hpp | 20 ++++++ core/api/service/internal/internal_api.hpp | 32 +++++++++ .../internal/internal_jrpc_processor.cpp | 28 ++++++++ .../internal/internal_jrpc_processor.hpp | 29 ++++++++ .../internal/requests/set_log_level.hpp | 35 +++++++++ core/injector/CMakeLists.txt | 3 +- core/injector/application_injector.cpp | 7 +- core/log/logger.cpp | 72 +++++++++++-------- core/log/logger.hpp | 14 ++-- 13 files changed, 242 insertions(+), 37 deletions(-) create mode 100644 core/api/service/internal/CMakeLists.txt create mode 100644 core/api/service/internal/impl/internal_api_impl.cpp create mode 100644 core/api/service/internal/impl/internal_api_impl.hpp create mode 100644 core/api/service/internal/internal_api.hpp create mode 100644 core/api/service/internal/internal_jrpc_processor.cpp create mode 100644 core/api/service/internal/internal_jrpc_processor.hpp create mode 100644 core/api/service/internal/requests/set_log_level.hpp diff --git a/core/api/service/CMakeLists.txt b/core/api/service/CMakeLists.txt index eb9a7e6e02..5bd17b333e 100644 --- a/core/api/service/CMakeLists.txt +++ b/core/api/service/CMakeLists.txt @@ -21,3 +21,4 @@ add_subdirectory(state) add_subdirectory(system) add_subdirectory(rpc) add_subdirectory(payment) +add_subdirectory(internal) diff --git a/core/api/service/chain/chain_api.hpp b/core/api/service/chain/chain_api.hpp index dc29cfb2d1..fc33b6c91d 100644 --- a/core/api/service/chain/chain_api.hpp +++ b/core/api/service/chain/chain_api.hpp @@ -18,7 +18,7 @@ namespace kagome::api { class ApiService; /** - * @class ChainApi privides interface for blockchain api + * @class ChainApi provides interface for blockchain api */ class ChainApi { public: diff --git a/core/api/service/internal/CMakeLists.txt b/core/api/service/internal/CMakeLists.txt new file mode 100644 index 0000000000..16d8cf46eb --- /dev/null +++ b/core/api/service/internal/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Copyright Soramitsu Co., Ltd. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(internal_api_service + internal_jrpc_processor.cpp + impl/internal_api_impl.cpp + ) +target_link_libraries(internal_api_service + buffer + api_service + ) diff --git a/core/api/service/internal/impl/internal_api_impl.cpp b/core/api/service/internal/impl/internal_api_impl.cpp new file mode 100644 index 0000000000..ec2c2c586f --- /dev/null +++ b/core/api/service/internal/impl/internal_api_impl.cpp @@ -0,0 +1,23 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "api/service/internal/impl/internal_api_impl.hpp" + +#include "log/logger.hpp" + +namespace kagome::api { + + outcome::result InternalApiImpl::setLogLevel( + const std::string &group, const std::string &level_str) { + OUTCOME_TRY(level, log::str2lvl(level_str)); + + if (not log::setLevelOfGroup(group, level)) { + return log::Error::WRONG_GROUP; + } + + return outcome::success(); + } + +} // namespace kagome::api diff --git a/core/api/service/internal/impl/internal_api_impl.hpp b/core/api/service/internal/impl/internal_api_impl.hpp new file mode 100644 index 0000000000..310c68e3a2 --- /dev/null +++ b/core/api/service/internal/impl/internal_api_impl.hpp @@ -0,0 +1,20 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_API_INTERNALAPIIMPL +#define KAGOME_API_INTERNALAPIIMPL + +#include "api/service/internal/internal_api.hpp" + +namespace kagome::api { + + class InternalApiImpl : public InternalApi { + public: + outcome::result setLogLevel(const std::string &group, + const std::string &level) override; + }; +} // namespace kagome::api + +#endif // KAGOME_API_INTERNALAPIIMPL diff --git a/core/api/service/internal/internal_api.hpp b/core/api/service/internal/internal_api.hpp new file mode 100644 index 0000000000..2d1790ee81 --- /dev/null +++ b/core/api/service/internal/internal_api.hpp @@ -0,0 +1,32 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_API_INTERNALAPI +#define KAGOME_API_INTERNALAPI + +#include + +#include "common/buffer.hpp" +#include "outcome/outcome.hpp" +#include "primitives/block_data.hpp" +#include "primitives/block_header.hpp" +#include "primitives/common.hpp" + +namespace kagome::api { + + /** + * @class InternalApi provides doing various action over RPC + */ + class InternalApi { + public: + virtual ~InternalApi() = default; + + virtual outcome::result setLogLevel(const std::string &group, + const std::string &level) = 0; + }; + +} // namespace kagome::api + +#endif // KAGOME_API_INTERNALAPI diff --git a/core/api/service/internal/internal_jrpc_processor.cpp b/core/api/service/internal/internal_jrpc_processor.cpp new file mode 100644 index 0000000000..d4c39aeecf --- /dev/null +++ b/core/api/service/internal/internal_jrpc_processor.cpp @@ -0,0 +1,28 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "api/service/internal/internal_jrpc_processor.hpp" + +#include "api/jrpc/jrpc_method.hpp" +#include "api/service/internal/requests/set_log_level.hpp" + +namespace kagome::api::internal { + + InternalJrpcProcessor::InternalJrpcProcessor( + std::shared_ptr server, std::shared_ptr api) + : api_{std::move(api)}, server_{std::move(server)} { + BOOST_ASSERT(api_ != nullptr); + BOOST_ASSERT(server_ != nullptr); + } + + template + using Handler = Method; + + void InternalJrpcProcessor::registerHandlers() { + server_->registerHandler("internal_setLogLevel", + Handler(api_)); + } + +} // namespace kagome::api::internal diff --git a/core/api/service/internal/internal_jrpc_processor.hpp b/core/api/service/internal/internal_jrpc_processor.hpp new file mode 100644 index 0000000000..fe23811d2c --- /dev/null +++ b/core/api/service/internal/internal_jrpc_processor.hpp @@ -0,0 +1,29 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_API_INTERNAL_INTERNALJRPCPROCESSOR +#define KAGOME_API_INTERNAL_INTERNALJRPCPROCESSOR + +#include "api/jrpc/jrpc_processor.hpp" +#include "api/jrpc/jrpc_server_impl.hpp" +#include "api/service/internal/internal_api.hpp" + +namespace kagome::api::internal { + /** + * @brief extrinsic submission service implementation + */ + class InternalJrpcProcessor : public JRpcProcessor { + public: + InternalJrpcProcessor(std::shared_ptr server, + std::shared_ptr api); + void registerHandlers() override; + + private: + std::shared_ptr api_; + std::shared_ptr server_; + }; +} // namespace kagome::api::internal + +#endif // KAGOME_API_INTERNAL_INTERNALJRPCPROCESSOR diff --git a/core/api/service/internal/requests/set_log_level.hpp b/core/api/service/internal/requests/set_log_level.hpp new file mode 100644 index 0000000000..48dabcb7b5 --- /dev/null +++ b/core/api/service/internal/requests/set_log_level.hpp @@ -0,0 +1,35 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_API_INTERNAL_REQUEST_SETLOGLEVEL +#define KAGOME_API_INTERNAL_REQUEST_SETLOGLEVEL + +#include "api/service/base_request.hpp" + +#include "log/logger.hpp" + +namespace kagome::api::internal::request { + + struct SetLogLevel final + : details::RequestType> { + SetLogLevel(std::shared_ptr &api) : api_(api){}; + + outcome::result execute() override { + const auto &group = + getParam<1>().has_value() ? getParam<0>() : log::defaultGroupName; + + const auto &level = + getParam<1>().has_value() ? getParam<1>().value() : getParam<0>(); + + return api_->setLogLevel(group, level); + } + + private: + std::shared_ptr api_; + }; + +} // namespace kagome::api::internal::request + +#endif // KAGOME_API_INTERNAL_REQUEST_SETLOGLEVEL diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index 3dbd1d8adc..17b78ae10c 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -89,8 +89,6 @@ target_link_libraries(application_injector state_protocol_observer storage_code_provider sync_protocol_observer - system_api_service - system_api_service telemetry tagged_transaction_queue_api transaction_payment_api @@ -104,6 +102,7 @@ target_link_libraries(application_injector system_api_service payment_api_service rpc_api_service + internal_api_service kademlia_storage_backend peer_manager babe_util diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 68d5ba4584..ccd89a96a1 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -24,6 +24,8 @@ #include "api/service/child_state/child_state_jrpc_processor.hpp" #include "api/service/child_state/impl/child_state_api_impl.hpp" #include "api/service/impl/api_service_impl.hpp" +#include "api/service/internal/impl/internal_api_impl.hpp" +#include "api/service/internal/internal_jrpc_processor.hpp" #include "api/service/payment/impl/payment_api_impl.hpp" #include "api/service/payment/payment_jrpc_processor.hpp" #include "api/service/rpc/impl/rpc_api_impl.hpp" @@ -1137,7 +1139,9 @@ namespace { injector.template create< std::shared_ptr>(), injector.template create< - std::shared_ptr>()}; + std::shared_ptr>(), + injector.template create< + std::shared_ptr>()}; return api::ApiServiceImpl::ProcessorSpan{processors}; }), // bind interfaces @@ -1328,6 +1332,7 @@ namespace { [](auto const &injector) { return get_recovery_mode(injector); }), di::bind.template to(), di::bind.template to(), + di::bind.template to(), // user-defined overrides... std::forward(args)...); diff --git a/core/log/logger.cpp b/core/log/logger.cpp index f6349a93f5..56c46a1edf 100644 --- a/core/log/logger.cpp +++ b/core/log/logger.cpp @@ -6,10 +6,24 @@ #include #include #include +#include #include "log/logger.hpp" #include "log/profiling_logger.hpp" +OUTCOME_CPP_DEFINE_CATEGORY(kagome::log, Error, e) { + using E = kagome::log::Error; + switch (e) { + case E::WRONG_LEVEL: + return "Unknown level"; + case E::WRONG_GROUP: + return "Unknown group"; + case E::WRONG_LOGGER: + return "Unknown logger"; + } + return "Unknown log::Error"; +} + namespace kagome::log { namespace { @@ -22,29 +36,29 @@ namespace kagome::log { "Logging system is not ready. " "kagome::log::setLoggingSystem() must be executed once before"); } + } // namespace - outcome::result str2lvl(std::string_view str) { - if (str == "trace") { - return Level::TRACE; - } else if (str == "debug") { - return Level::DEBUG; - } else if (str == "verbose") { - return Level::VERBOSE; - } else if (str == "info" or str == "inf") { - return Level::INFO; - } else if (str == "warning" or str == "warn") { - return Level::WARN; - } else if (str == "error" or str == "err") { - return Level::ERROR; - } else if (str == "critical" or str == "crit") { - return Level::CRITICAL; - } else if (str == "off" or str == "no") { - return Level::OFF; - } else { - return boost::system::error_code{}; - } + outcome::result str2lvl(std::string_view str) { + if (str == "trace") { + return Level::TRACE; + } else if (str == "debug") { + return Level::DEBUG; + } else if (str == "verbose") { + return Level::VERBOSE; + } else if (str == "info" or str == "inf") { + return Level::INFO; + } else if (str == "warning" or str == "warn") { + return Level::WARN; + } else if (str == "error" or str == "err") { + return Level::ERROR; + } else if (str == "critical" or str == "crit") { + return Level::CRITICAL; + } else if (str == "off" or str == "no") { + return Level::OFF; + } else { + return Error::WRONG_LEVEL; } - } // namespace + } void setLoggingSystem( std::shared_ptr logging_system) { @@ -115,22 +129,22 @@ namespace kagome::log { ->getLogger(tag, group, level); } - void setLevelOfGroup(const std::string &group_name, Level level) { + bool setLevelOfGroup(const std::string &group_name, Level level) { ensure_logger_system_is_initialized(); - logging_system_->setLevelOfGroup(group_name, level); + return logging_system_->setLevelOfGroup(group_name, level); } - void resetLevelOfGroup(const std::string &group_name) { + bool resetLevelOfGroup(const std::string &group_name) { ensure_logger_system_is_initialized(); - logging_system_->resetLevelOfGroup(group_name); + return logging_system_->resetLevelOfGroup(group_name); } - void setLevelOfLogger(const std::string &logger_name, Level level) { + bool setLevelOfLogger(const std::string &logger_name, Level level) { ensure_logger_system_is_initialized(); - logging_system_->setLevelOfLogger(logger_name, level); + return logging_system_->setLevelOfLogger(logger_name, level); } - void resetLevelOfLogger(const std::string &logger_name) { + bool resetLevelOfLogger(const std::string &logger_name) { ensure_logger_system_is_initialized(); - logging_system_->resetLevelOfLogger(logger_name); + return logging_system_->resetLevelOfLogger(logger_name); } } // namespace kagome::log diff --git a/core/log/logger.hpp b/core/log/logger.hpp index a0ad6b9675..d263dcc802 100644 --- a/core/log/logger.hpp +++ b/core/log/logger.hpp @@ -22,6 +22,10 @@ namespace kagome::log { using Level = soralog::Level; using Logger = std::shared_ptr; + enum class Error : uint8_t { WRONG_LEVEL = 1, WRONG_GROUP, WRONG_LOGGER }; + + outcome::result str2lvl(std::string_view str); + void setLoggingSystem(std::shared_ptr logging_system); void tuneLoggingSystem(const std::vector &cfg); @@ -37,11 +41,11 @@ namespace kagome::log { const std::string &group, Level level); - void setLevelOfGroup(const std::string &group_name, Level level); - void resetLevelOfGroup(const std::string &group_name); + bool setLevelOfGroup(const std::string &group_name, Level level); + bool resetLevelOfGroup(const std::string &group_name); - void setLevelOfLogger(const std::string &logger_name, Level level); - void resetLevelOfLogger(const std::string &logger_name); + bool setLevelOfLogger(const std::string &logger_name, Level level); + bool resetLevelOfLogger(const std::string &logger_name); template Ret format_arg(T const &t) { @@ -146,4 +150,6 @@ namespace kagome::log { } // namespace kagome::log +OUTCOME_HPP_DECLARE_ERROR(kagome::log, Error); + #endif // KAGOME_LOGGER_HPP From 9516996188c57f8fcc078716f1311ec7b94d541c Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 5 Aug 2022 19:15:43 +0300 Subject: [PATCH 16/74] feature: change logging level over RPC Signed-off-by: Dmitriy Khaustov aka xDimon --- core/utils/CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/utils/CMakeLists.txt b/core/utils/CMakeLists.txt index c1816c3d2d..5d366b5d65 100644 --- a/core/utils/CMakeLists.txt +++ b/core/utils/CMakeLists.txt @@ -3,8 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 # -add_executable(storage_explorer storage_explorer.cpp) -target_link_libraries(storage_explorer application_injector) +add_executable(storage_explorer + storage_explorer.cpp + ) +target_link_libraries(storage_explorer + application_injector + ) add_executable(kagome-db-editor kagome_db_editor.cpp From 9481e76a2b54c0ac172d5a9f6a1f8e18325a345c Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 02:47:55 +0300 Subject: [PATCH 17/74] feature: accelerate hasDirectChain check for case of finalized chain Signed-off-by: Dmitriy Khaustov aka xDimon --- core/blockchain/impl/block_tree_impl.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index db74725a9f..6d5aa86d3d 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1177,6 +1177,29 @@ namespace kagome::blockchain { } // else, we need to use a database + + // Try to use optimal way, if ancestor and descendant in the finalized chain + if (descendant_depth <= getLastFinalized().number) { + auto res = header_repo_->getHashByNumber(descendant_depth); + BOOST_ASSERT_MSG(res.has_value(), + "Any finalized block must be accessible by number"); + // Check if descendant in finalised chain + if (res.value() == descendant) { + res = header_repo_->getHashByNumber(ancestor_depth); + BOOST_ASSERT_MSG(res.has_value(), + "Any finalized block must be accessible by number"); + if (res.value() == ancestor) { + // Ancestor and descendant in the finalized chain, + // therefore they have direct chain between each other + return true; + } else { + // Ancestor in the finalized chain, but descendant is not, + // therefore they can not have direct chain between each other + return false; + } + } + } + auto current_hash = descendant; while (current_hash != ancestor) { auto current_header_res = header_repo_->getBlockHeader(current_hash); From 70b5877a877b8ef284151749724c9c0506d11245 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 02:49:43 +0300 Subject: [PATCH 18/74] refactor: optimize prepareFromGenesis of AuthorityManager Signed-off-by: Dmitriy Khaustov aka xDimon --- .../authority/impl/authority_manager_impl.cpp | 99 ++++++++++++------- .../authority/impl/authority_manager_impl.hpp | 5 +- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index c23a14236d..7053221cf4 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -5,14 +5,13 @@ #include "consensus/authority/impl/authority_manager_impl.hpp" -#include -#include #include #include #include #include "application/app_state_manager.hpp" +#include "blockchain/block_header_repository.hpp" #include "blockchain/block_tree.hpp" #include "common/visitor.hpp" #include "consensus/authority/authority_manager_error.hpp" @@ -32,16 +31,19 @@ namespace kagome::authority { AuthorityManagerImpl::AuthorityManagerImpl( Config config, std::shared_ptr app_state_manager, + std::shared_ptr header_repo, std::shared_ptr block_tree, std::shared_ptr trie_storage, std::shared_ptr grandpa_api, std::shared_ptr hasher) : config_{std::move(config)}, + header_repo_(std::move(header_repo)), block_tree_(std::move(block_tree)), trie_storage_(std::move(trie_storage)), grandpa_api_(std::move(grandpa_api)), hasher_(std::move(hasher)), log_{log::createLogger("AuthorityManager", "authority")} { + BOOST_ASSERT(header_repo_ != nullptr); BOOST_ASSERT(block_tree_ != nullptr); BOOST_ASSERT(grandpa_api_ != nullptr); BOOST_ASSERT(trie_storage_ != nullptr); @@ -851,60 +853,83 @@ namespace kagome::authority { bool AuthorityManagerImpl::prepareFromGenesis() { auto t_start = std::chrono::high_resolution_clock::now(); - std::vector> - headers; + const auto finalized_block = block_tree_->getLastFinalized(); - auto header = block_tree_->getBlockHeader(finalized_block.hash); - headers.emplace_back(finalized_block.hash, header.value()); - size_t count1 = 0; - while (header.has_value()) { - auto hash = header.value().parent_hash; - header = block_tree_->getBlockHeader(hash); - if (header.has_value()) { - if (not(++count1 % 10000)) { - SL_WARN(log_, "{} headers loaded", count1); - } - headers.emplace_back(hash, header.value()); - } - } root_ = authority::ScheduleNode::createAsRoot( - {headers.back().second.number, headers.back().first}); - auto authorities = grandpa_api_->authorities(headers.back().first).value(); + {0, block_tree_->getGenesisBlockHash()}); + + auto authorities = + grandpa_api_->authorities(block_tree_->getGenesisBlockHash()).value(); authorities.id = 0; root_->actual_authorities = std::make_shared(std::move(authorities)); - count1 = 0; - size_t count2 = 0; - for (const auto &[hash, header] : boost::range::reverse(headers)) { - if (not(++count1 % 1000)) { - SL_WARN(log_, "{} digests applied ({})", count1, count2); - count2 = 0; + size_t count = 0; + size_t delta = 0; + + for (consensus::grandpa::BlockNumber block_number = 1; + block_number <= finalized_block.number; + ++block_number) { + auto hash_res = header_repo_->getHashByNumber(block_number); + if (hash_res.has_error()) { + SL_CRITICAL(log_, + "Can't get hash of block #{} of the best chain: {}", + block_number, + hash_res.error()); + return false; } + const auto &block_hash = hash_res.value(); + + primitives::BlockInfo block_info(block_number, block_hash); + + auto header_res = block_tree_->getBlockHeader(block_hash); + if (header_res.has_error()) { + SL_CRITICAL(log_, + "Can't get header of block {}: {}", + block_info, + header_res.error()); + return false; + } + const auto &header = header_res.value(); + + bool found = false; for (const auto &digest_item : header.digest) { std::ignore = visit_in_place( digest_item, - [&, num = header.number, hash = hash]( - const primitives::Consensus &consensus_message) - -> outcome::result { - ++count2; - return onConsensus(primitives::BlockInfo{num, hash}, - consensus_message); + [&](const primitives::Consensus &msg) -> outcome::result { + if (msg.consensus_engine_id == primitives::kGrandpaEngineId) { + ++count; + ++delta; + found = true; + } + return onConsensus(block_info, msg); }, [](const auto &) { return outcome::success(); }); } - if (not(count1 % 10000)) { - prune(primitives::BlockInfo{header.number, hash}); + if (not(block_number % 512)) { + prune(block_info); + } + + if (found) { + SL_VERBOSE(log_, + "{} headers are observed and {} digests are applied (+{})", + block_number, + count, + delta); + delta = 0; } } auto t_end = std::chrono::high_resolution_clock::now(); - log_->warn( - "Applied authorities within {} ms", - std::chrono::duration_cast(t_end - t_start) - .count()); + SL_INFO(log_, + "{} headers are observed and {} digests are applied within {}s", + finalized_block.number, + count, + std::chrono::duration_cast(t_end - t_start) + .count()); + return true; } diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 36da79dd49..29f521d9ea 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -20,7 +20,8 @@ namespace kagome::authority { } namespace kagome::blockchain { class BlockTree; -} + class BlockHeaderRepository; +} // namespace kagome::blockchain namespace kagome::primitives { struct AuthorityList; struct BabeConfiguration; @@ -50,6 +51,7 @@ namespace kagome::authority { AuthorityManagerImpl( Config config, std::shared_ptr app_state_manager, + std::shared_ptr header_repo, std::shared_ptr block_tree, std::shared_ptr trie_storage, std::shared_ptr grandpa_api, @@ -123,6 +125,7 @@ namespace kagome::authority { bool prepareFromGenesis(); Config config_; + std::shared_ptr header_repo_; std::shared_ptr block_tree_; std::shared_ptr trie_storage_; std::shared_ptr grandpa_api_; From db4d41cf700f2db0f72a1958f88ff834a972d06b Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 02:52:28 +0300 Subject: [PATCH 19/74] refactor: preload a bit more blocks while fast syncing Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/synchronizer_impl.cpp | 94 +++++++++++++++---------- core/network/impl/synchronizer_impl.hpp | 5 ++ 2 files changed, 63 insertions(+), 36 deletions(-) diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index e57789bcfb..c461602631 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -1123,7 +1123,12 @@ namespace kagome::network { } ancestry_.erase(hash); - if (known_blocks_.size() < kMinPreloadedBlockAmount) { + auto minPreloadedBlockAmount = + sync_method_ == application::AppConfiguration::SyncMethod::Full + ? kMinPreloadedBlockAmount + : kMinPreloadedBlockAmountForFastSyncing; + + if (known_blocks_.size() < minPreloadedBlockAmount) { SL_TRACE(log_, "{} blocks in queue: ask next portion of block", known_blocks_.size()); @@ -1268,11 +1273,12 @@ namespace kagome::network { continue; } + primitives::BlockInfo block_info(g_it->first, hash); + auto &peers = b_it->second.peers; if (peers.empty()) { - SL_TRACE(log_, - "Block {} don't have any peer. Go to next one", - primitives::BlockInfo(g_it->first, hash)); + SL_TRACE( + log_, "Block {} don't have any peer. Go to next one", block_info); continue; } @@ -1306,45 +1312,61 @@ namespace kagome::network { res.error().message()); return; } - SL_DEBUG( - self->log_, "Portion of blocks from {} is loaded", peer_id); + SL_DEBUG(self->log_, + "Portion of blocks from {} is loaded till {}", + peer_id, + res.value()); + if (self->known_blocks_.empty()) { + self->askNextPortionOfBlocks(); + } } }; - auto lower = generations_.begin()->first; - auto upper = generations_.rbegin()->first + 1; - auto hint = generations_.rbegin()->first; + if (sync_method_ == application::AppConfiguration::SyncMethod::Full) { + auto lower = generations_.begin()->first; + auto upper = generations_.rbegin()->first + 1; + auto hint = generations_.rbegin()->first; - SL_DEBUG(log_, - "Start to find common block with {} in #{}..#{} to fill queue", - peer_id, - generations_.begin()->first, - generations_.rbegin()->first); - findCommonBlock( - peer_id, - lower, - upper, - hint, - [wp = weak_from_this(), peer_id, handler = std::move(handler)]( - outcome::result res) { - if (auto self = wp.lock()) { - if (not res.has_value()) { + SL_DEBUG( + log_, + "Start to find common block with {} in #{}..#{} to fill queue", + peer_id, + generations_.begin()->first, + generations_.rbegin()->first); + findCommonBlock( + peer_id, + lower, + upper, + hint, + [wp = weak_from_this(), peer_id, handler = std::move(handler)]( + outcome::result res) { + if (auto self = wp.lock()) { + if (not res.has_value()) { + SL_DEBUG(self->log_, + "Can't load next portion of blocks from {}: {}", + peer_id, + res.error().message()); + handler(res); + return; + } + auto &common_block_info = res.value(); SL_DEBUG(self->log_, - "Can't load next portion of blocks from {}: {}", + "Start to load next portion of blocks from {} " + "since block {}", peer_id, - res.error().message()); - handler(res); - return; + common_block_info); + self->loadBlocks( + peer_id, common_block_info, std::move(handler)); } - auto &block_info = res.value(); - SL_DEBUG(self->log_, - "Start to load next portion of blocks from {} " - "since block {}", - peer_id, - block_info); - self->loadBlocks(peer_id, block_info, std::move(handler)); - } - }); + }); + } else { + SL_DEBUG(log_, + "Start to load next portion of blocks from {} " + "since block {}", + peer_id, + block_info); + loadBlocks(peer_id, block_info, std::move(handler)); + } return; } diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 6cd0742259..29a6757ba5 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -46,6 +46,11 @@ namespace kagome::network { /// 256 is doubled max amount block in BlocksResponse. static constexpr size_t kMinPreloadedBlockAmount = 256; + /// Block amount enough for applying and preloading other ones + /// simultaneously during fast syncing + static constexpr size_t kMinPreloadedBlockAmountForFastSyncing = + kMinPreloadedBlockAmount * 5; + /// Indicating how far the block can be subscribed to. /// In general we don't needed wait very far blocks. This limit to avoid /// extra memory consumption. From af658506d51a875a0920031e5fc6c3649427ee77 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 03:11:44 +0300 Subject: [PATCH 20/74] fix: logging some outcome error Signed-off-by: Dmitriy Khaustov aka xDimon --- .../authority/impl/authority_manager_impl.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 7053221cf4..bedbaedee1 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -251,13 +251,13 @@ namespace kagome::authority { auto &&set_id_opt_res = fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header); if (set_id_opt_res.has_error()) { - auto &error = set_id_opt_res.error(); log_->warn( - "Couldn't fetch authority set id from trie storage for block #{} " - "({}): {}. Recalculating from genesis.", - header.number, - hash, - error.message()); + "Can't fetch authority set id from trie storage for block {}: {}", + primitives::BlockInfo(header.number, hash), + set_id_opt_res.error().message()); + log_->info( + "Recalculating from genesis " + "(going to take a few dozens of seconds)"); return prepareFromGenesis(); } auto &set_id_opt = set_id_opt_res.value(); @@ -876,7 +876,7 @@ namespace kagome::authority { SL_CRITICAL(log_, "Can't get hash of block #{} of the best chain: {}", block_number, - hash_res.error()); + hash_res.error().message()); return false; } const auto &block_hash = hash_res.value(); @@ -888,7 +888,7 @@ namespace kagome::authority { SL_CRITICAL(log_, "Can't get header of block {}: {}", block_info, - header_res.error()); + header_res.error().message()); return false; } const auto &header = header_res.value(); From d02c0bf20cd905364c9d2af3e3c92b9a83e659fb Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 03:28:46 +0300 Subject: [PATCH 21/74] feature: smart processing of duplicating blocks during fast sync Signed-off-by: Dmitriy Khaustov aka xDimon --- .../babe/impl/block_appender_impl.cpp | 82 +++++++++---------- .../babe/impl/block_appender_impl.hpp | 2 + 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 821b713819..3cb12977a5 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -8,16 +8,13 @@ #include #include "blockchain/block_tree_error.hpp" -#include "blockchain/impl/common.hpp" #include "consensus/babe/consistency_keeper.hpp" #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" #include "consensus/babe/types/slot.hpp" -#include "consensus/grandpa/impl/voting_round_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "primitives/common.hpp" #include "scale/scale.hpp" -#include "transaction_pool/transaction_pool_error.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus, BlockAppenderImpl::Error, e) { using E = kagome::consensus::BlockAppenderImpl::Error; @@ -74,13 +71,29 @@ namespace kagome::consensus { primitives::BlockInfo block_info(header.number, block_hash); - if (auto header_res = block_tree_->getBlockHeader(header.parent_hash); - header_res.has_error() - && header_res.error() == blockchain::BlockTreeError::HEADER_NOT_FOUND) { - logger_->warn("Skipping a block {} with unknown parent", block_info); - return Error::PARENT_NOT_FOUND; - } else if (header_res.has_error()) { - return header_res.as_failure(); + if (last_appended_.has_value()) { + if (last_appended_->number > block_info.number) { + SL_DEBUG( + logger_, "Skip early appended header of block: {}", block_info); + return outcome::success(); + } + if (last_appended_.value() == block_info) { + SL_DEBUG(logger_, "Skip just appended header of block: {}", block_info); + return outcome::success(); + } + } + + if (last_appended_ + != primitives::BlockInfo(header.number - 1, header.parent_hash)) { + if (auto header_res = block_tree_->getBlockHeader(header.parent_hash); + header_res.has_error() + && header_res.error() + == blockchain::BlockTreeError::HEADER_NOT_FOUND) { + logger_->warn("Skipping a block {} with unknown parent", block_info); + return Error::PARENT_NOT_FOUND; + } else if (header_res.has_error()) { + return header_res.as_failure(); + } } // get current time to measure performance if block execution @@ -205,15 +218,14 @@ namespace kagome::consensus { digest_item, [&](const primitives::Consensus &consensus_message) -> outcome::result { - return authority_update_observer_->onConsensus( - primitives::BlockInfo{block.header.number, block_hash}, - consensus_message); + return authority_update_observer_->onConsensus(block_info, + consensus_message); }, [](const auto &) { return outcome::success(); }); if (res.has_error()) { SL_ERROR(logger_, "Error while processing consensus digests of block {}: {}", - block_hash, + block_info, res.error().message()); return res.as_failure(); } @@ -224,53 +236,33 @@ namespace kagome::consensus { if (b.justification.has_value()) { SL_VERBOSE(logger_, "Justification received for block {}", block_info); - // try to apply left in justification store values first - if (not justifications_.empty()) { - std::vector to_remove; - for (const auto &[block_justified_for, justification] : - justifications_) { - auto res = applyJustification(block_justified_for, justification); - if (res) { - to_remove.push_back(block_justified_for); - } - } - if (not to_remove.empty()) { - for (const auto &item : to_remove) { - justifications_.erase(item); - } - } - } - auto res = applyJustification(block_info, b.justification.value()); - if (res.has_error()) { - if (res - == outcome::failure(grandpa::VotingRoundError::NOT_ENOUGH_WEIGHT)) { - justifications_.emplace(block_info, b.justification.value()); - } else { - return res.as_failure(); - } - } else { - // safely could be remove if current justification applied successfully - justifications_.clear(); + SL_ERROR(logger_, + "Error while applying of block {} justification: {}", + block_info, + res.error().message()); + return res.as_failure(); } } SL_DEBUG(logger_, - "Imported header of block {} within {} ms", + "Imported header of block {} within {} us", block_info, - std::chrono::duration_cast( + std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - t_start) .count()); if (block_info.number % 512 == 0) { SL_INFO(logger_, - "Imported another 512 headers of blocks. Last one imported of {}", - block_info); + "Imported another 512 headers of blocks. Last one imported is {}", + block_info); } consistency_guard.commit(); + last_appended_.emplace(std::move(block_info)); + return outcome::success(); } diff --git a/core/consensus/babe/impl/block_appender_impl.hpp b/core/consensus/babe/impl/block_appender_impl.hpp index 9f331352f1..2419f96b34 100644 --- a/core/consensus/babe/impl/block_appender_impl.hpp +++ b/core/consensus/babe/impl/block_appender_impl.hpp @@ -60,6 +60,8 @@ namespace kagome::consensus { std::shared_ptr babe_util_; std::shared_ptr consistency_keeper_; + std::optional last_appended_; + // Justification Store for Future Applying std::map justifications_; From 8f2c6d7d57015f69273e328f6bda01fa45764de3 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 18:35:05 +0300 Subject: [PATCH 22/74] feature: indicate average speed of appending while fast syncing Signed-off-by: Dmitriy Khaustov aka xDimon --- .../babe/impl/block_appender_impl.cpp | 27 ++++++++++++------- .../babe/impl/block_appender_impl.hpp | 5 ++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 3cb12977a5..26e3348497 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -246,17 +246,26 @@ namespace kagome::consensus { } } - SL_DEBUG(logger_, - "Imported header of block {} within {} us", - block_info, - std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - t_start) - .count()); + auto now = std::chrono::high_resolution_clock::now(); + + SL_DEBUG( + logger_, + "Imported header of block {} within {} us", + block_info, + std::chrono::duration_cast(now - t_start) + .count()); - if (block_info.number % 512 == 0) { + auto block_delta = block_info.number - speed_data_.block_number; + auto time_delta = now - speed_data_.time; + if (block_delta >= 25000 or time_delta >= std::chrono::minutes(1)) { SL_INFO(logger_, - "Imported another 512 headers of blocks. Last one imported is {}", - block_info); + "Imported {} more headers of blocks. Average speed is {} bps", + block_delta, + block_delta + / std::chrono::duration_cast(time_delta) + .count()); + speed_data_.block_number = block_info.number; + speed_data_.time = now; } consistency_guard.commit(); diff --git a/core/consensus/babe/impl/block_appender_impl.hpp b/core/consensus/babe/impl/block_appender_impl.hpp index 2419f96b34..b03fee7c9f 100644 --- a/core/consensus/babe/impl/block_appender_impl.hpp +++ b/core/consensus/babe/impl/block_appender_impl.hpp @@ -65,6 +65,11 @@ namespace kagome::consensus { // Justification Store for Future Applying std::map justifications_; + struct { + std::chrono::high_resolution_clock::time_point time; + primitives::BlockNumber block_number; + } speed_data_; + log::Logger logger_; }; From 908f4ae1a4b9e5db3b386eb2fedecd18708f415b Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 18:36:40 +0300 Subject: [PATCH 23/74] feature: down rating of peer at disconnection Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/stream_engine.hpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/core/network/impl/stream_engine.hpp b/core/network/impl/stream_engine.hpp index 1123c0b1fc..e98bebfb81 100644 --- a/core/network/impl/stream_engine.hpp +++ b/core/network/impl/stream_engine.hpp @@ -20,6 +20,7 @@ #include "network/helpers/peer_id_formatter.hpp" #include "network/helpers/scale_message_read_writer.hpp" #include "network/protocol_base.hpp" +#include "network/rating_repository.hpp" #include "subscription/subscriber.hpp" #include "subscription/subscription_engine.hpp" #include "utils/safe_object.hpp" @@ -42,6 +43,9 @@ namespace kagome::network { using Stream = libp2p::connection::Stream; using StreamEnginePtr = std::shared_ptr; + static constexpr auto kDownVoteByDisconnectionExpirationTimeout = + std::chrono::seconds(30); + enum class Direction : uint8_t { INCOMING = 1, OUTGOING = 2, @@ -121,7 +125,9 @@ namespace kagome::network { StreamEngine &operator=(StreamEngine &&) = delete; ~StreamEngine() = default; - StreamEngine() : logger_{log::createLogger("StreamEngine", "network")} {} + StreamEngine(std::shared_ptr peer_rating_repository) + : peer_rating_repository_(std::move(peer_rating_repository)), + logger_{log::createLogger("StreamEngine", "network")} {} template static StreamEnginePtr create(Args &&...args) { @@ -488,7 +494,10 @@ namespace kagome::network { if (stream_res == outcome::failure( std::make_error_code(std::errc::not_connected))) { - self->del(peer_id); + self->peer_rating_repository_->updateForATime( + peer_id, + -1000, + kDownVoteByDisconnectionExpirationTimeout); return; } @@ -545,7 +554,9 @@ namespace kagome::network { }); } + std::shared_ptr peer_rating_repository_; log::Logger logger_; + SafeObject streams_; }; From c7e40b4aaba36f202c2b0a824643b5c7c5682b06 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 18:37:31 +0300 Subject: [PATCH 24/74] tune: preload more headers while fast syncing Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/synchronizer_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 29a6757ba5..64afead9bb 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -49,7 +49,7 @@ namespace kagome::network { /// Block amount enough for applying and preloading other ones /// simultaneously during fast syncing static constexpr size_t kMinPreloadedBlockAmountForFastSyncing = - kMinPreloadedBlockAmount * 5; + kMinPreloadedBlockAmount * 40; /// Indicating how far the block can be subscribed to. /// In general we don't needed wait very far blocks. This limit to avoid From bef9c7031a1fd8288a9e24428fdb53af78e662fb Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 19:13:39 +0300 Subject: [PATCH 25/74] clean: remove commented piece of code Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/peer_manager_impl.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index b2ccc85ca8..c5e2a458ae 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -212,22 +212,6 @@ namespace kagome::network { disconnectFromPeer(peer_id); } -// // Check if disconnected -// auto block_announce_protocol = router_->getBlockAnnounceProtocol(); -// BOOST_ASSERT_MSG(block_announce_protocol, -// "Router did not provide block announce protocol"); -// -// for (auto it = active_peers_.begin(); it != active_peers_.end();) { -// auto [peer_id, data] = *it++; -// // TODO(d.khaustov) consider better alive check logic -// if (not stream_engine_->isAlive(peer_id, block_announce_protocol)) { -// // Found disconnected -// const auto &peer_id_ref = peer_id; -// SL_DEBUG(log_, "Found dead peer: {}", peer_id_ref); -// disconnectFromPeer(peer_id); -// } -// } -// // Soft limit is exceeded if (active_peers_.size() > soft_limit) { // Get oldest peer From a984c526961633df25fc4c00dca969d8a2872d95 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Wed, 10 Aug 2022 19:15:19 +0300 Subject: [PATCH 26/74] fix: test Signed-off-by: Dmitriy Khaustov aka xDimon --- test/core/consensus/authority/authority_manager_test.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index 529d2d8c4f..c07911783d 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -9,6 +9,7 @@ #include "consensus/authority/impl/schedule_node.hpp" #include "mock/core/application/app_state_manager_mock.hpp" +#include "mock/core/blockchain/block_header_repository_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/runtime/grandpa_api_mock.hpp" @@ -53,6 +54,8 @@ class AuthorityManagerTest : public testing::Test { authorities->emplace_back(makeAuthority("GenesisAuthority2", 10)); authorities->emplace_back(makeAuthority("GenesisAuthority3", 15)); + header_repo = std::make_shared(); + block_tree = std::make_shared(); storage = std::make_shared(); @@ -77,6 +80,7 @@ class AuthorityManagerTest : public testing::Test { authority_manager = std::make_shared(AuthorityManagerImpl::Config{}, app_state_manager, + header_repo, block_tree, storage, grandpa_api, @@ -148,6 +152,7 @@ class AuthorityManagerTest : public testing::Test { const std::vector leaves{genesis_block.hash}; std::shared_ptr app_state_manager; + std::shared_ptr header_repo; std::shared_ptr block_tree; std::shared_ptr storage; std::shared_ptr grandpa_api; From 8a4ef62ba92ed782376e6a239339fe79da387a1c Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Thu, 11 Aug 2022 11:38:04 +0300 Subject: [PATCH 27/74] fix: allow recovery to block without state Signed-off-by: Dmitriy Khaustov aka xDimon --- core/blockchain/impl/block_tree_impl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 6d5aa86d3d..b1a2b5fadf 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -404,9 +404,11 @@ namespace kagome::blockchain { // Check if target block has state if (auto res = trie_storage->getEphemeralBatchAt(state_root); res.has_error()) { - SL_CRITICAL( + SL_WARN( log, "Can't get state of target block: {}", res.error().message()); - return res.as_failure(); + SL_CRITICAL( + log, + "You will need to use `--sync Fast' CLI arg the next time you start"); } for (auto it = block_tree_leaves.rbegin(); it != block_tree_leaves.rend(); From b233deb661b79b6651ef7aa43ba3430b0b69fa08 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Thu, 11 Aug 2022 11:39:39 +0300 Subject: [PATCH 28/74] fix: fallback way to process justification with votes for future blocks Signed-off-by: Dmitriy Khaustov aka xDimon --- .../babe/impl/block_appender_impl.cpp | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 26e3348497..6fec590597 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -12,6 +12,7 @@ #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" #include "consensus/babe/types/slot.hpp" +#include "consensus/grandpa/impl/voting_round_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "primitives/common.hpp" #include "scale/scale.hpp" @@ -236,13 +237,38 @@ namespace kagome::consensus { if (b.justification.has_value()) { SL_VERBOSE(logger_, "Justification received for block {}", block_info); + // try to apply left in justification store values first + if (not justifications_.empty()) { + std::vector to_remove; + for (const auto &[block_justified_for, justification] : + justifications_) { + auto res = applyJustification(block_justified_for, justification); + if (res) { + to_remove.push_back(block_justified_for); + } + } + if (not to_remove.empty()) { + for (const auto &item : to_remove) { + justifications_.erase(item); + } + } + } + auto res = applyJustification(block_info, b.justification.value()); if (res.has_error()) { - SL_ERROR(logger_, - "Error while applying of block {} justification: {}", - block_info, - res.error().message()); - return res.as_failure(); + if (res + == outcome::failure(grandpa::VotingRoundError::NOT_ENOUGH_WEIGHT)) { + justifications_.emplace(block_info, b.justification.value()); + } else { + SL_ERROR(logger_, + "Error while applying of block {} justification: {}", + block_info, + res.error().message()); + return res.as_failure(); + } + } else { + // safely could be remove if current justification applied successfully + justifications_.clear(); } } @@ -257,7 +283,7 @@ namespace kagome::consensus { auto block_delta = block_info.number - speed_data_.block_number; auto time_delta = now - speed_data_.time; - if (block_delta >= 25000 or time_delta >= std::chrono::minutes(1)) { + if (block_delta >= 20000 or time_delta >= std::chrono::minutes(1)) { SL_INFO(logger_, "Imported {} more headers of blocks. Average speed is {} bps", block_delta, From 21e1267d070ce21147834a1a2cc90532846a4dd1 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 12 Aug 2022 19:30:33 +0300 Subject: [PATCH 29/74] fix: typo Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/stream_engine.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/network/impl/stream_engine.hpp b/core/network/impl/stream_engine.hpp index e98bebfb81..71743c8e61 100644 --- a/core/network/impl/stream_engine.hpp +++ b/core/network/impl/stream_engine.hpp @@ -66,7 +66,7 @@ namespace kagome::network { } outgoing; std::deque)>> - deffered_messages; + deferred_messages; public: explicit ProtocolDescr(std::shared_ptr proto) @@ -464,7 +464,7 @@ namespace kagome::network { logger_->debug("DUMP: I={} O={} Messages:{}", descr.incoming.stream, descr.outgoing.stream, - descr.deffered_messages.size()); + descr.deferred_messages.size()); } }); logger_->debug("DUMP: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"); @@ -504,7 +504,7 @@ namespace kagome::network { self->streams_.exclusiveAccess([&](auto &streams) { self->forSubscriber( peer_id, streams, protocol, [&](auto, auto &descr) { - descr.deffered_messages.clear(); + descr.deferred_messages.clear(); descr.dropReserved(); }); }); @@ -524,10 +524,10 @@ namespace kagome::network { Direction::OUTGOING); descr.dropReserved(); - while (!descr.deffered_messages.empty()) { - auto &msg = descr.deffered_messages.front(); + while (!descr.deferred_messages.empty()) { + auto &msg = descr.deferred_messages.front(); msg(stream); - descr.deffered_messages.pop_front(); + descr.deferred_messages.pop_front(); } }); BOOST_ASSERT(existing); @@ -542,7 +542,7 @@ namespace kagome::network { std::shared_ptr msg) { streams_.exclusiveAccess([&](auto &streams) { forSubscriber(peer_id, streams, protocol, [&](auto, auto &descr) { - descr.deffered_messages.push_back( + descr.deferred_messages.push_back( [wp(weak_from_this()), peer_id, protocol, msg(std::move(msg))]( std::shared_ptr stream) { if (auto self = wp.lock()) { From bb80b7f714b9a94dabc3709b63a7bbb0d6d7efee Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 12 Aug 2022 19:46:28 +0300 Subject: [PATCH 30/74] refactor: assert Signed-off-by: Dmitriy Khaustov aka xDimon --- core/runtime/runtime_api/impl/core.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/runtime/runtime_api/impl/core.cpp b/core/runtime/runtime_api/impl/core.cpp index bffd3bc176..2114b3d5b3 100644 --- a/core/runtime/runtime_api/impl/core.cpp +++ b/core/runtime/runtime_api/impl/core.cpp @@ -34,8 +34,11 @@ namespace kagome::runtime { outcome::result CoreImpl::execute_block( const primitives::Block &block) { - OUTCOME_TRY(parent, header_repo_->getBlockHeader(block.header.parent_hash)); - BOOST_ASSERT(parent.number == block.header.number - 1); + BOOST_ASSERT([] { + auto parent_res = header_repo_->getBlockHeader(block.header.parent_hash); + return parent_res.has_value() + and parent_res.value().number == block.header.number - 1; + }()); changes_tracker_->onBlockExecutionStart(block.header.parent_hash); OUTCOME_TRY(executor_->persistentCallAt( block.header.parent_hash, "Core_execute_block", block)); From c55804efe34e3ae6cc534287f95b7104e648f124 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 12 Aug 2022 19:49:00 +0300 Subject: [PATCH 31/74] feature: additional state of babe for syncing Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/babe.hpp | 4 +- core/consensus/babe/impl/babe_impl.cpp | 155 ++++++++++++++++++------- core/consensus/babe/impl/babe_impl.hpp | 15 ++- core/injector/application_injector.cpp | 13 ++- 4 files changed, 138 insertions(+), 49 deletions(-) diff --git a/core/consensus/babe/babe.hpp b/core/consensus/babe/babe.hpp index fdecf2721c..e4105da5e8 100644 --- a/core/consensus/babe/babe.hpp +++ b/core/consensus/babe/babe.hpp @@ -23,8 +23,10 @@ namespace kagome::consensus::babe { ~Babe() override = default; enum class State { - WAIT_REMOTE_STATUS, // Node is just executed and waits status of remote + WAIT_REMOTE_STATUS, // Node is just launched and waits status of remote // peer to sync missing blocks + HEADERS_LOADING, // Fast sync requested; phase of headers downloading + STATE_LOADING, // Fast sync requested; phase of state downloading CATCHING_UP, // Node recognized the missing blocks and started fetching // blocks between the best missing one and one of the // available ones diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index ba1d6e4db2..239f237fc2 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -8,6 +8,7 @@ #include #include +#include "application/app_configuration.hpp" #include "blockchain/block_storage_error.hpp" #include "blockchain/block_tree_error.hpp" #include "common/buffer.hpp" @@ -18,6 +19,7 @@ #include "consensus/babe/types/babe_block_header.hpp" #include "consensus/babe/types/seal.hpp" #include "network/block_announce_transmitter.hpp" +#include "network/helpers/peer_id_formatter.hpp" #include "network/synchronizer.hpp" #include "network/types/block_announce.hpp" #include "primitives/inherent_data.hpp" @@ -33,7 +35,10 @@ namespace { using namespace std::literals::chrono_literals; namespace kagome::consensus::babe { + using SyncMethod = application::AppConfiguration::SyncMethod; + BabeImpl::BabeImpl( + const application::AppConfiguration &app_config, std::shared_ptr app_state_manager, std::shared_ptr lottery, std::shared_ptr configuration, @@ -51,7 +56,7 @@ namespace kagome::consensus::babe { std::shared_ptr synchronizer, std::shared_ptr babe_util, std::shared_ptr offchain_worker_api) - : was_synchronized_{false}, + : app_config_(app_config), lottery_{std::move(lottery)}, babe_configuration_{std::move(configuration)}, proposer_{std::move(proposer)}, @@ -138,7 +143,23 @@ namespace kagome::consensus::babe { return false; } } - current_state_ = State::WAIT_REMOTE_STATUS; + + switch (app_config_.syncMethod()) { + case SyncMethod::Full: + current_state_ = State::WAIT_REMOTE_STATUS; + break; + + case SyncMethod::Fast: + // Has uncompleted downloading state + if (false) { // FIXME + // Start loading of state + current_state_ = State::STATE_LOADING; + } else { + current_state_ = State::HEADERS_LOADING; + } + break; + } + return true; } @@ -264,6 +285,12 @@ namespace kagome::consensus::babe { void BabeImpl::onRemoteStatus(const libp2p::peer::PeerId &peer_id, const network::Status &status) { + // If state is loading, just to ping of loading + if (current_state_ == Babe::State::STATE_LOADING) { + startStateSyncing(peer_id); + return; + } + const auto &last_finalized_block = block_tree_->getLastFinalized(); auto current_best_block_res = @@ -286,6 +313,12 @@ namespace kagome::consensus::babe { void BabeImpl::onBlockAnnounce(const libp2p::peer::PeerId &peer_id, const network::BlockAnnounce &announce) { + // If state is loading, just to ping of loading + if (current_state_ == Babe::State::STATE_LOADING) { + startStateSyncing(peer_id); + return; + } + const auto &last_finalized_block = block_tree_->getLastFinalized(); auto current_best_block_res = @@ -323,79 +356,113 @@ namespace kagome::consensus::babe { if (block_res.has_error()) { return; } + const auto &block = block_res.value(); + // Headers are loaded; Start to sync state + if (self->current_state_ == Babe::State::HEADERS_LOADING) { + self->startStateSyncing(peer_id); + return; + } + + // Just synced if (self->current_state_ == Babe::State::CATCHING_UP) { - const auto &block = block_res.value(); - self->synchronizer_->syncState( - peer_id, - block, - {common::Buffer()}, - [self, block = block, peer_id]( - outcome::result - block_res) mutable { - if (block_res.has_error()) { - SL_WARN(self->log_, - "Block result error: {}", - block_res.error().message()); - return; - } - - SL_INFO(self->log_, - "Catching up is finished on block {}", - block_res.value()); - self->current_state_ = Babe::State::SYNCHRONIZED; - self->was_synchronized_ = true; - self->telemetry_->notifyWasSynchronized(); - }); + SL_INFO(self->log_, "Catching up is finished on block {}", block); + self->current_state_ = Babe::State::SYNCHRONIZED; + self->was_synchronized_ = true; + self->telemetry_->notifyWasSynchronized(); } - self->onSynchronized(); - // Propagate announce - self->block_announce_transmitter_->blockAnnounce( - std::move(announce)); + // Synced + if (self->current_state_ == Babe::State::SYNCHRONIZED) { + self->onSynchronized(); + // Propagate announce + self->block_announce_transmitter_->blockAnnounce( + std::move(announce)); + return; + } } }); } void BabeImpl::startCatchUp(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &target_block) { - // synchronize missing blocks with their bodies + BOOST_ASSERT(current_state_ != Babe::State::HEADERS_LOADING); + BOOST_ASSERT(current_state_ != Babe::State::STATE_LOADING); + // synchronize missing blocks with their bodies auto is_ran = synchronizer_->syncByBlockInfo( target_block, peer_id, - [wp = weak_from_this(), bn = target_block.number, peer_id]( + [wp = weak_from_this(), block = target_block, peer_id]( outcome::result res) { if (auto self = wp.lock()) { self->synchronizer_->endSync(); if (res.has_error()) { SL_DEBUG(self->log_, - "Catching up to block #{} on {} is failed: {}", - bn, - peer_id.toBase58(), + "Catching up {} to block {} is failed: {}", + peer_id, + block, res.error().message()); return; } - SL_DEBUG( - self->log_, - "Catching up to block #{} on {} is going (on block #{} now)", - bn, - peer_id.toBase58(), - res.value().number); + SL_DEBUG(self->log_, + "Catching up {} to block {} is going; on block {} now", + peer_id, + block, + res.value()); } }, false); if (is_ran) { - SL_VERBOSE(log_, - "Catching up to block #{} on {} is ran", - target_block.number, - peer_id.toBase58()); + SL_VERBOSE( + log_, "Catching up {} to block {} is ran", peer_id, target_block); current_state_ = State::CATCHING_UP; } } + void BabeImpl::startStateSyncing(const libp2p::peer::PeerId &peer_id) { + BOOST_ASSERT(current_state_ == Babe::State::HEADERS_LOADING + or current_state_ == Babe::State::STATE_LOADING); + if (current_state_ == Babe::State::HEADERS_LOADING + or current_state_ == Babe::State::STATE_LOADING) { + SL_WARN(log_, "Syncing of state can not be start: Bad state of babe"); + return; + } + + current_state_ = Babe::State::STATE_LOADING; + + // Switch to last finalized to have a state on it + auto block_at_state = block_tree_->getLastFinalized(); + + SL_TRACE(log_, + "Trying to sync state on block {} from {}", + block_at_state, + peer_id); + + synchronizer_->syncState( + peer_id, + block_at_state, + {{}}, + [wp = weak_from_this(), block_at_state, peer_id](auto res) mutable { + if (auto self = wp.lock()) { + if (res.has_error()) { + SL_WARN(self->log_, + "Syncing of state on block {} is failed: {}", + block_at_state, + res.error().message()); + return; + } + + SL_INFO(self->log_, + "State on block {} is synced successfully", + block_at_state); + self->current_state_ = Babe::State::CATCHING_UP; + } + }); + } + void BabeImpl::onSynchronized() { // won't start block production without keypair if (not keypair_) { @@ -505,7 +572,7 @@ namespace kagome::consensus::babe { continue; } if (best_block_.number == 0) { - // Only genesis may not have a babe digest + // Only genesis block header might not have a babe digest break; } BOOST_ASSERT(babe_digests_res.has_value()); diff --git a/core/consensus/babe/impl/babe_impl.hpp b/core/consensus/babe/impl/babe_impl.hpp index 53b1193112..4c2683cd21 100644 --- a/core/consensus/babe/impl/babe_impl.hpp +++ b/core/consensus/babe/impl/babe_impl.hpp @@ -31,6 +31,10 @@ #include "storage/trie/trie_storage.hpp" #include "telemetry/service.hpp" +namespace kagome::application { + class AppConfiguration; +} + namespace kagome::network { class Synchronizer; class BlockAnnounceTransmitter; @@ -38,7 +42,7 @@ namespace kagome::network { namespace kagome::runtime { class OffchainWorkerApi; -} // namespace kagome::runtime +} namespace kagome::consensus::babe { @@ -60,7 +64,8 @@ namespace kagome::consensus::babe { /** * Create an instance of Babe implementation */ - BabeImpl(std::shared_ptr app_state_manager, + BabeImpl(const application::AppConfiguration &app_config, + std::shared_ptr app_state_manager, std::shared_ptr lottery, std::shared_ptr configuration, std::shared_ptr proposer, @@ -111,6 +116,8 @@ namespace kagome::consensus::babe { void startCatchUp(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &target_block); + void startStateSyncing(const libp2p::peer::PeerId &peer_id); + void runSlot(); /** @@ -148,7 +155,7 @@ namespace kagome::consensus::babe { bool isSecondarySlotsAllowed() const; - bool was_synchronized_; + const application::AppConfiguration &app_config_; std::shared_ptr lottery_; std::shared_ptr babe_configuration_; std::shared_ptr proposer_; @@ -168,6 +175,8 @@ namespace kagome::consensus::babe { State current_state_{State::WAIT_REMOTE_STATUS}; + bool was_synchronized_{false}; + std::atomic_bool active_{false}; EpochDescriptor current_epoch_; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 2334cbc234..dec11019ce 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -430,15 +430,25 @@ namespace { if (initialized) { return initialized.value(); } + + auto log = log::createLogger("Injector", "injector"); + auto configuration_res = babe_api->configuration(block_hash); if (not configuration_res) { + if (configuration_res + == outcome::failure(runtime::RuntimeEnvironmentFactory::Error:: + FAILED_TO_SET_STORAGE_STATE)) { + SL_CRITICAL(log, + "State for block {} has not found. " + "Try to launch with `--sync Fast' CLI arg", + block_hash); + } common::raise(configuration_res.error()); } auto configuration = std::make_shared( std::move(configuration_res.value())); - auto log = log::createLogger("Injector", "injector"); for (const auto &authority : configuration->genesis_authorities) { SL_DEBUG(log, "Babe authority: {:l}", authority.id.id); } @@ -1387,6 +1397,7 @@ namespace { auto session_keys = injector.template create>(); initialized = std::make_shared( + injector.template create(), injector.template create>(), injector.template create>(), injector.template create>(), From 84f362bb6439d0aaa8fa406591152f217ab98f1b Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Fri, 12 Aug 2022 19:50:00 +0300 Subject: [PATCH 32/74] clean: unused code Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/synchronizer_new_impl.cpp | 678 -------------------- core/network/impl/synchronizer_new_impl.hpp | 193 ------ 2 files changed, 871 deletions(-) delete mode 100644 core/network/impl/synchronizer_new_impl.cpp delete mode 100644 core/network/impl/synchronizer_new_impl.hpp diff --git a/core/network/impl/synchronizer_new_impl.cpp b/core/network/impl/synchronizer_new_impl.cpp deleted file mode 100644 index 281325bd37..0000000000 --- a/core/network/impl/synchronizer_new_impl.cpp +++ /dev/null @@ -1,678 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "network/impl/synchronizer_new_impl.hpp" - -#include - -#include "application/app_configuration.hpp" -#include "blockchain/block_tree_error.hpp" -#include "network/helpers/peer_id_formatter.hpp" -#include "network/types/block_attributes.hpp" -#include "primitives/common.hpp" -#include "storage/changes_trie/changes_tracker.hpp" -#include "storage/trie/serialization/trie_serializer.hpp" -#include "storage/trie/trie_batches.hpp" -#include "storage/trie/trie_storage.hpp" - -OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerNewImpl::Error, e) { - using E = kagome::network::SynchronizerNewImpl::Error; - switch (e) { - case E::SHUTTING_DOWN: - return "Node is shutting down"; - case E::EMPTY_RESPONSE: - return "Response is empty"; - case E::RESPONSE_WITHOUT_BLOCK_HEADER: - return "Response does not contain header of some block"; - case E::RESPONSE_WITHOUT_BLOCK_BODY: - return "Response does not contain body of some block"; - case E::DISCARDED_BLOCK: - return "Block is discarded"; - case E::WRONG_ORDER: - return "Wrong order of blocks/headers in response"; - case E::INVALID_HASH: - return "Hash does not match"; - case E::ALREADY_IN_QUEUE: - return "Block is already enqueued"; - case E::PEER_BUSY: - return "Peer is busy"; - case E::ARRIVED_TOO_EARLY: - return "Block is arrived too early. Try to process it late"; - case E::DUPLICATE_REQUEST: - return "Duplicate of recent request has been detected"; - } - return "unknown error"; -} - -namespace { - constexpr const char *kImportQueueLength = - "kagome_import_queue_blocks_submitted"; - - kagome::network::BlockAttributes attributesForSync( - kagome::application::AppConfiguration::SyncMethod method) { - using SM = kagome::application::AppConfiguration::SyncMethod; - switch (method) { - case SM::Full: - return kagome::network::BlocksRequest::kBasicAttributes; - case SM::Fast: - return kagome::network::BlockAttribute::HEADER - | kagome::network::BlockAttribute::JUSTIFICATION; - } - return kagome::network::BlocksRequest::kBasicAttributes; - } -} // namespace - -namespace kagome::network { - - SynchronizerNewImpl::SynchronizerNewImpl( - const application::AppConfiguration &app_config, - std::shared_ptr app_state_manager, - std::shared_ptr block_tree, - std::shared_ptr changes_tracker, - std::shared_ptr block_appender, - std::shared_ptr block_executor, - std::shared_ptr serializer, - std::shared_ptr storage, - std::shared_ptr router, - std::shared_ptr scheduler, - std::shared_ptr hasher) - : block_tree_(std::move(block_tree)), - trie_changes_tracker_(std::move(changes_tracker)), - block_appender_(std::move(block_appender)), - block_executor_(std::move(block_executor)), - serializer_(std::move(serializer)), - storage_(std::move(storage)), - router_(std::move(router)), - scheduler_(std::move(scheduler)), - hasher_(std::move(hasher)) { - BOOST_ASSERT(block_tree_); - BOOST_ASSERT(trie_changes_tracker_ != nullptr); - BOOST_ASSERT(block_executor_); - BOOST_ASSERT(serializer_); - BOOST_ASSERT(storage_); - BOOST_ASSERT(router_); - BOOST_ASSERT(scheduler_); - BOOST_ASSERT(hasher_); - - BOOST_ASSERT(app_state_manager); - - sync_method_ = app_config.syncMethod(); - - // Register metrics - metrics_registry_->registerGaugeFamily( - kImportQueueLength, "Number of blocks submitted to the import queue"); - metric_import_queue_length_ = - metrics_registry_->registerGaugeMetric(kImportQueueLength); - metric_import_queue_length_->set(0); - - app_state_manager->atShutdown([this] { node_is_shutting_down_ = true; }); - } - - bool SynchronizerNewImpl::subscribeToBlock( - const primitives::BlockInfo &block_info, SyncResultHandler &&handler) { - // Check if block is already in tree - if (block_tree_->hasBlockHeader(block_info.hash)) { - scheduler_->schedule( - [handler = std::move(handler), block_info] { handler(block_info); }); - return false; - } - - auto last_finalized_block = block_tree_->getLastFinalized(); - // Check if block from discarded side-chain - if (last_finalized_block.number <= block_info.number) { - scheduler_->schedule( - [handler = std::move(handler)] { handler(Error::DISCARDED_BLOCK); }); - return false; - } - - // Check if block has arrived too early - auto best_block_res = - block_tree_->getBestContaining(last_finalized_block.hash, std::nullopt); - BOOST_ASSERT(best_block_res.has_value()); - const auto &best_block = best_block_res.value(); - if (best_block.number + kMaxDistanceToBlockForSubscription - < block_info.number) { - scheduler_->schedule([handler = std::move(handler)] { - handler(Error::ARRIVED_TOO_EARLY); - }); - return false; - } - - subscriptions_.emplace(block_info, std::move(handler)); - return true; - } - - void SynchronizerNewImpl::notifySubscribers( - const primitives::BlockInfo &block, outcome::result res) { - auto range = subscriptions_.equal_range(block); - for (auto it = range.first; it != range.second;) { - auto cit = it++; - if (auto node = subscriptions_.extract(cit)) { - if (res.has_error()) { - auto error = res.as_failure(); - scheduler_->schedule( - [handler = std::move(node.mapped()), error] { handler(error); }); - } else { - scheduler_->schedule( - [handler = std::move(node.mapped()), block] { handler(block); }); - } - } - } - } - - bool SynchronizerNewImpl::syncByBlockInfo( - const primitives::BlockInfo &block_info, - const libp2p::peer::PeerId &peer_id, - Synchronizer::SyncResultHandler &&handler, - bool subscribe_to_block) { - if (not state_syncing_.load() && not syncing_.load()) { - syncing_.store(true); - } else { - return false; - } - // Subscribe on demand - if (subscribe_to_block) { - subscribeToBlock(block_info, std::move(handler)); - } - - // If provided block is already enqueued, just remember peer - if (auto it = known_blocks_.find(block_info.hash); - it != known_blocks_.end()) { - auto &block_in_queue = it->second; - block_in_queue.peers.emplace(peer_id); - if (handler) handler(block_info); - return false; - } - - const auto &last_finalized_block = block_tree_->getLastFinalized(); - - auto best_block_res = - block_tree_->getBestContaining(last_finalized_block.hash, std::nullopt); - BOOST_ASSERT(best_block_res.has_value()); - const auto &best_block = best_block_res.value(); - - // Provided block is equal our best one. Nothing needs to do. - if (block_info == best_block) { - if (handler) handler(block_info); - return false; - } - - loadBlocks(peer_id, last_finalized_block, std::move(handler)); - - return true; - } - - bool SynchronizerNewImpl::syncByBlockHeader( - const primitives::BlockHeader &header, - const libp2p::peer::PeerId &peer_id, - Synchronizer::SyncResultHandler &&handler) { - if (not state_syncing_.load() && not syncing_.load()) { - syncing_.store(true); - } else { - return false; - } - auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); - const primitives::BlockInfo block_info(header.number, block_hash); - - // Block was applied before - if (block_tree_->getBlockHeader(block_hash).has_value()) { - return false; - } - - // Block is already enqueued - if (auto it = known_blocks_.find(block_info.hash); - it != known_blocks_.end()) { - auto &block_in_queue = it->second; - block_in_queue.peers.emplace(peer_id); - return false; - } - - loadBlocks(peer_id, block_info, std::move(handler)); - - return true; - } - - void SynchronizerNewImpl::loadBlocks(const libp2p::peer::PeerId &peer_id, - primitives::BlockInfo from, - SyncResultHandler &&handler) { - // Interrupts process if node is shutting down - if (node_is_shutting_down_) { - if (handler) handler(Error::SHUTTING_DOWN); - return; - } - - network::BlocksRequest request{attributesForSync(sync_method_), - from.hash, - std::nullopt, - network::Direction::ASCENDING, - std::nullopt}; - - auto response_handler = [wp = weak_from_this(), - from, - peer_id, - handler = std::move(handler), - parent_hash = primitives::BlockHash{}]( - auto &&response_res) mutable { - auto self = wp.lock(); - if (not self) { - return; - } - - // Any error interrupts loading of blocks - if (response_res.has_error()) { - SL_ERROR(self->log_, - "Can't load blocks from {} beginning block {}: {}", - peer_id, - from, - response_res.error().message()); - if (handler) handler(response_res.as_failure()); - return; - } - auto &blocks = response_res.value().blocks; - - // No block in response is abnormal situation. - // At least one starting block should be returned as existing - if (blocks.empty()) { - SL_ERROR(self->log_, - "Can't load blocks from {} beginning block {}: " - "Response does not have any blocks", - peer_id, - from); - if (handler) handler(Error::EMPTY_RESPONSE); - return; - } - - SL_TRACE(self->log_, - "{} blocks are loaded from {} beginning block {}", - blocks.size(), - peer_id, - from); - - primitives::BlockInfo last_loaded_block; - - for (auto &block : blocks) { - // Check if header is provided - if (not block.header.has_value()) { - SL_ERROR(self->log_, - "Can't load blocks from {} starting from block {}: " - "Received block without header", - peer_id, - from); - if (handler) handler(Error::RESPONSE_WITHOUT_BLOCK_HEADER); - return; - } - - auto &header = block.header.value(); - - const auto &last_finalized_block = - self->block_tree_->getLastFinalized(); - - // Check by number if block is not finalized yet - if (last_finalized_block.number >= header.number) { - if (last_finalized_block.number == header.number) { - if (last_finalized_block.hash != block.hash) { - SL_ERROR(self->log_, - "Can't load blocks from {} starting from block {}: " - "Received discarded block {}", - peer_id, - from, - BlockInfo(header.number, block.hash)); - if (handler) handler(Error::DISCARDED_BLOCK); - return; - } - - SL_TRACE(self->log_, - "Skip block {} received from {}: " - "it is finalized with block #{}", - BlockInfo(header.number, block.hash), - peer_id, - last_finalized_block.number); - continue; - } - - SL_TRACE(self->log_, - "Skip block {} received from {}: " - "it is below the last finalized block #{}", - BlockInfo(header.number, block.hash), - peer_id, - last_finalized_block.number); - continue; - } - - // Check if block is not discarded - if (last_finalized_block.number + 1 == header.number) { - if (last_finalized_block.hash != header.parent_hash) { - SL_ERROR(self->log_, - "Can't complete blocks loading from {} starting from " - "block {}: Received discarded block {}", - peer_id, - from, - BlockInfo(header.number, header.parent_hash)); - if (handler) handler(Error::DISCARDED_BLOCK); - return; - } - - // Start to check parents - parent_hash = header.parent_hash; - } - - // Check if block is in chain - static const primitives::BlockHash zero_hash; - if (parent_hash != header.parent_hash && parent_hash != zero_hash) { - SL_ERROR(self->log_, - "Can't complete blocks loading from {} starting from " - "block {}: Received block is not descendant of previous", - peer_id, - from); - if (handler) handler(Error::WRONG_ORDER); - return; - } - - // Check if hash is valid - auto calculated_hash = - self->hasher_->blake2b_256(scale::encode(header).value()); - if (block.hash != calculated_hash) { - SL_ERROR(self->log_, - "Can't complete blocks loading from {} starting from " - "block {}: " - "Received block whose hash does not match the header", - peer_id, - from); - if (handler) handler(Error::INVALID_HASH); - return; - } - - last_loaded_block = {header.number, block.hash}; - - parent_hash = block.hash; - - // Add block in queue and save peer or just add peer for existing record - auto it = self->known_blocks_.find(block.hash); - if (it == self->known_blocks_.end()) { - self->known_blocks_.emplace(block.hash, KnownBlock{block, {peer_id}}); - self->metric_import_queue_length_->set(self->known_blocks_.size()); - } else { - it->second.peers.emplace(peer_id); - SL_TRACE(self->log_, - "Skip block {} received from {}: already enqueued", - BlockInfo(header.number, block.hash), - peer_id); - continue; - } - - SL_TRACE(self->log_, - "Enqueue block {} received from {}", - BlockInfo(header.number, block.hash), - peer_id); - - self->generations_.emplace(header.number, block.hash); - self->ancestry_.emplace(header.parent_hash, block.hash); - } - - if (from.number + 20 >= last_loaded_block.number || blocks.size() < 127) { - self->applyNextBlock(); - handler(last_loaded_block); - return; - } - self->scheduler_->schedule([self, - peer_id, - handler = std::move(handler), - last_loaded_block]() mutable { - self->applyNextBlock(); - self->loadBlocks(peer_id, last_loaded_block, std::move(handler)); - }); - }; - - auto protocol = router_->getSyncProtocol(); - BOOST_ASSERT_MSG(protocol, "Router did not provide sync protocol"); - protocol->request(peer_id, std::move(request), std::move(response_handler)); - } - - void SynchronizerNewImpl::syncState(const libp2p::peer::PeerId &peer_id, - const primitives::BlockInfo &block, - const std::vector &keys, - SyncResultHandler &&handler) { - if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { - // execute if not already started or iteration continues - if (not state_syncing_.load() - || (not keys.empty() && not keys[0].empty())) { - state_syncing_.store(true); - if (keys[0].empty()) { - sync_block_ = block; - } - network::StateRequest request{block.hash, keys, true}; - - auto protocol = router_->getStateProtocol(); - BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); - - SL_TRACE(log_, "State syncing started."); - - auto response_handler = [wp = weak_from_this(), - block, - peer_id, - handler = std::move(handler)]( - auto &&response_res) mutable { - auto self = wp.lock(); - if (not self) { - return; - } - - if (response_res.has_error()) { - SL_WARN(self->log_, - "State syncing failed with error: {}", - response_res.error().message()); - if (handler) handler(response_res.as_failure()); - return; - } - - for (unsigned i = 0; i < response_res.value().entries.size(); ++i) { - const auto &response = response_res.value().entries[i]; - - // get or create batch - auto batch = - self->batches_store_.count(response.state_root) - ? std::get<2>(self->batches_store_[response.state_root]) - : self->storage_ - ->getPersistentBatchAt( - self->serializer_->getEmptyRootHash()) - .value(); - - // main storage entries size empty at child storage state syncing - if (response.entries.size()) { - SL_TRACE(self->log_, - "Syncing {}th item. Current key {}. Keys received {}.", - i, - response.entries[0].key.toHex(), - response.entries.size()); - for (const auto &entry : response.entries) { - std::ignore = batch->put(entry.key, entry.value); - } - - // store batch to continue at next response - if (!response.complete) { - self->batches_store_[response.state_root] = { - response.entries.back().key, i, batch}; - } else { - self->batches_store_.erase(response.state_root); - } - } - - if (response.complete) { - auto res = batch->commit(); - SL_INFO( - self->log_, - "{} syncing finished. Root hash: {}. {}.", - i ? "Child state" : "State", - res.value().toHex(), - res.value() == response.state_root ? "Match" : "Don't match"); - if (res.value() != response.state_root) { - SL_INFO( - self->log_, "Should be {}", response.state_root.toHex()); - } - self->trie_changes_tracker_->onBlockAdded(block.hash); - } - - // just calculate state entries in main storage for trace log - if (!i) { - self->entries_ += response.entries.size(); - } - } - - // not well formed way to place 0th batch key to front - std::map keymap; - for (const auto &[_, val] : self->batches_store_) { - unsigned i = std::get<1>(val); - keymap[i] = std::get<0>(val); - SL_TRACE(self->log_, "Index: {}, Key: {}", i, keymap[i]); - } - - std::vector keys; - for (const auto &[_, val] : keymap) { - keys.push_back(val); - } - - if (response_res.value().entries[0].complete) { - self->sync_method_ = - application::AppConfiguration::SyncMethod::Full; - if (handler) { - handler(block); - self->state_syncing_.store(false); - } - } else { - SL_TRACE(self->log_, - "State syncing continues. {} entries loaded", - self->entries_); - self->syncState(peer_id, block, keys, std::move(handler)); - } - }; - - protocol->request( - peer_id, std::move(request), std::move(response_handler)); - } - } else { - if (handler) { - handler(block); - } - } - } - - void SynchronizerNewImpl::applyNextBlock() { - if (generations_.empty()) { - SL_TRACE(log_, "No block for applying"); - return; - } - - primitives::BlockHash hash; - - while (true) { - auto generation_node = generations_.extract(generations_.begin()); - if (generation_node) { - hash = generation_node.mapped(); - break; - } - if (generations_.empty()) { - SL_TRACE(log_, "No block for applying"); - return; - } - } - - auto node = known_blocks_.extract(hash); - if (node) { - auto &block = node.mapped().data; - BOOST_ASSERT(block.header.has_value()); - auto number = block.header->number; - - const auto &last_finalized_block = block_tree_->getLastFinalized(); - - if (block.header->number <= last_finalized_block.number) { - auto header_res = block_tree_->getBlockHeader(hash); - if (not header_res.has_value()) { - auto n = discardBlock(block.hash); - SL_WARN( - log_, - "Block {} {} not applied as discarded", - BlockInfo(number, hash), - n ? fmt::format("and {} others have", n) : fmt::format("has")); - } - } else { - const BlockInfo block_info(block.header->number, block.hash); - - if (sync_method_ == application::AppConfiguration::SyncMethod::Full - && sync_block_ && block_info.number <= sync_block_.value().number) { - SL_WARN( - log_, "Skip {} till fast synchronized block", block_info.number); - applyNextBlock(); - } else { - auto applying_res = - sync_method_ == application::AppConfiguration::SyncMethod::Full - ? block_executor_->applyBlock(std::move(block)) - : block_appender_->appendBlock(std::move(block)); - - if (sync_method_ == application::AppConfiguration::SyncMethod::Full - && sync_block_ - && block_info.number == sync_block_.value().number + 1) { - sync_block_ = std::nullopt; - } - - notifySubscribers({number, hash}, applying_res); - - if (not applying_res.has_value()) { - if (applying_res - != outcome::failure(blockchain::BlockTreeError::BLOCK_EXISTS)) { - notifySubscribers({number, hash}, applying_res.as_failure()); - auto n = discardBlock(block_info.hash); - SL_WARN( - log_, - "Block {} {} been discarded: {}", - block_info, - n ? fmt::format("and {} others have", n) : fmt::format("has"), - applying_res.error().message()); - } else { - SL_DEBUG(log_, "Block {} is skipped as existing", block_info); - applyNextBlock(); - } - } else { - applyNextBlock(); - } - } - } - } - ancestry_.erase(hash); - - metric_import_queue_length_->set(known_blocks_.size()); - } - - size_t SynchronizerNewImpl::discardBlock( - const primitives::BlockHash &hash_of_discarding_block) { - std::queue queue; - queue.emplace(hash_of_discarding_block); - - size_t affected = 0; - while (not queue.empty()) { - const auto &hash = queue.front(); - - if (auto it = known_blocks_.find(hash); it != known_blocks_.end()) { - auto number = it->second.data.header->number; - notifySubscribers({number, hash}, Error::DISCARDED_BLOCK); - - known_blocks_.erase(it); - affected++; - } - - auto range = ancestry_.equal_range(hash); - for (auto it = range.first; it != range.second; ++it) { - queue.emplace(it->second); - } - ancestry_.erase(range.first, range.second); - - queue.pop(); - } - - metric_import_queue_length_->set(known_blocks_.size()); - - return affected; - } - -} // namespace kagome::network diff --git a/core/network/impl/synchronizer_new_impl.hpp b/core/network/impl/synchronizer_new_impl.hpp deleted file mode 100644 index d351225d4d..0000000000 --- a/core/network/impl/synchronizer_new_impl.hpp +++ /dev/null @@ -1,193 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef KAGOME_NETWORK_SYNCHRONIZERNEWIMPL -#define KAGOME_NETWORK_SYNCHRONIZERNEWIMPL - -#include "network/synchronizer.hpp" - -#include -#include - -#include - -#include "application/app_state_manager.hpp" -#include "consensus/babe/block_appender.hpp" -#include "consensus/babe/block_executor.hpp" -#include "metrics/metrics.hpp" -#include "network/router.hpp" -#include "storage/changes_trie/changes_tracker.hpp" - -namespace kagome::application { - class AppConfiguration; -} - -namespace kagome::storage::trie { - class PersistentTrieBatch; - class TrieSerializer; - class TrieStorage; -} // namespace kagome::storage::trie - -namespace kagome::storage::changes_trie { - class ChangesTracker; -} - -namespace kagome::network { - - class SynchronizerNewImpl - : public Synchronizer, - public std::enable_shared_from_this { - public: - /// Block amount enough for applying and preloading other ones - /// simultaneously. - /// 256 is doubled max amount block in BlocksResponse. - static constexpr size_t kMinPreloadedBlockAmount = 256; - - /// Indicating how far the block can be subscribed to. - /// In general we don't needed wait very far blocks. This limit to avoid - /// extra memory consumption. - static constexpr size_t kMaxDistanceToBlockForSubscription = - kMinPreloadedBlockAmount * 2; - - static constexpr std::chrono::milliseconds kRecentnessDuration = - std::chrono::seconds(60); - - enum class Error { - SHUTTING_DOWN = 1, - EMPTY_RESPONSE, - RESPONSE_WITHOUT_BLOCK_HEADER, - RESPONSE_WITHOUT_BLOCK_BODY, - DISCARDED_BLOCK, - WRONG_ORDER, - INVALID_HASH, - ALREADY_IN_QUEUE, - PEER_BUSY, - ARRIVED_TOO_EARLY, - DUPLICATE_REQUEST - }; - - SynchronizerNewImpl( - const application::AppConfiguration &app_config, - std::shared_ptr app_state_manager, - std::shared_ptr block_tree, - std::shared_ptr changes_tracker, - std::shared_ptr block_appender, - std::shared_ptr block_executor, - std::shared_ptr serializer, - std::shared_ptr storage, - std::shared_ptr router, - std::shared_ptr scheduler, - std::shared_ptr hasher); - - /// Enqueues loading (and applying) blocks from peer {@param peer_id} - /// since best common block up to provided {@param block_info}. - /// {@param handler} will be called when this process is finished or failed - /// @returns true if sync is ran (peer is not busy) - /// @note Is used for start/continue catching up. - bool syncByBlockInfo(const primitives::BlockInfo &block_info, - const libp2p::peer::PeerId &peer_id, - SyncResultHandler &&handler, - bool subscribe_to_block) override; - - /// Enqueues loading and applying block {@param block_info} from peer - /// {@param peer_id}. - /// @returns true if sync is ran (peer is not busy) - /// If provided block is the best after applying, {@param handler} be called - bool syncByBlockHeader(const primitives::BlockHeader &header, - const libp2p::peer::PeerId &peer_id, - SyncResultHandler &&handler) override; - - void syncState(const libp2p::peer::PeerId &peer_id, - const primitives::BlockInfo &block, - const std::vector &keys, - SyncResultHandler &&handler) override; - - /// Loads blocks from peer {@param peer_id} since block {@param from} till - /// its best. Calls {@param handler} when process is finished or failed - void loadBlocks(const libp2p::peer::PeerId &peer_id, - primitives::BlockInfo from, - SyncResultHandler &&handler); - - void endSync() override { - syncing_.store(false); - } - - private: - /// Subscribes handler for block with provided {@param block_info} - /// {@param handler} will be called When block is received or discarded - /// @returns true if subscription is successful - bool subscribeToBlock(const primitives::BlockInfo &block_info, - SyncResultHandler &&handler); - - /// Notifies subscribers about arrived block - void notifySubscribers(const primitives::BlockInfo &block_info, - outcome::result res); - - /// Pops next block from queue and tries to apply that - void applyNextBlock(); - - /// Removes block {@param block} and all all dependent on it from the queue - /// @returns number of affected blocks - size_t discardBlock(const primitives::BlockHash &block); - - std::shared_ptr block_tree_; - std::shared_ptr - trie_changes_tracker_; - std::shared_ptr block_appender_; - std::shared_ptr block_executor_; - std::shared_ptr serializer_; - std::shared_ptr storage_; - std::shared_ptr router_; - std::shared_ptr scheduler_; - std::shared_ptr hasher_; - application::AppConfiguration::SyncMethod sync_method_; - - // Metrics - metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); - metrics::Gauge *metric_import_queue_length_; - - log::Logger log_ = log::createLogger("Synchronizer", "synchronizer"); - - std::atomic_bool syncing_ = false; - std::atomic_bool state_syncing_ = false; - bool node_is_shutting_down_ = false; - - struct KnownBlock { - /// Data of block - primitives::BlockData data; - /// Peers who know this block - std::set peers; - }; - - // Already known (enqueued) but is not applied yet - std::unordered_map known_blocks_; - - std::optional sync_block_; - - // Blocks grouped by number - std::multimap generations_; - - // Links parent->child - std::unordered_multimap - ancestry_; - - std::multimap subscriptions_; - - std::atomic_bool applying_in_progress_ = false; - - std::unordered_map< - storage::trie::RootHash, - std::tuple>> - batches_store_; - size_t entries_{0}; - }; - -} // namespace kagome::network - -OUTCOME_HPP_DECLARE_ERROR(kagome::network, SynchronizerNewImpl::Error) - -#endif // KAGOME_NETWORK_SYNCHRONIZERNEWIMPL From b94a6020bff7548411223ed6aa5a46aad609945d Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 02:44:03 +0300 Subject: [PATCH 33/74] fix: test Signed-off-by: Dmitriy Khaustov aka xDimon --- test/core/consensus/babe/babe_test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/core/consensus/babe/babe_test.cpp b/test/core/consensus/babe/babe_test.cpp index d94179d519..620d0df81a 100644 --- a/test/core/consensus/babe/babe_test.cpp +++ b/test/core/consensus/babe/babe_test.cpp @@ -12,6 +12,7 @@ #include "consensus/babe/babe_error.hpp" #include "consensus/babe/impl/babe_impl.hpp" +#include "mock/core/application/app_configuration_mock.hpp" #include "mock/core/application/app_state_manager_mock.hpp" #include "mock/core/authorship/proposer_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" @@ -144,7 +145,8 @@ class BabeTest : public testing::Test { EXPECT_CALL(*sr25519_provider, sign(_, _)) .WillRepeatedly(Return(Sr25519Signature{})); - babe_ = std::make_shared(app_state_manager_, + babe_ = std::make_shared(app_config_, + app_state_manager_, lottery_, babe_config_, proposer_, @@ -174,6 +176,7 @@ class BabeTest : public testing::Test { .value(); } + application::AppConfigurationMock app_config_; std::shared_ptr app_state_manager_; std::shared_ptr lottery_; std::shared_ptr synchronizer_; From eebfd61c798372172fbfda8b7edc6b2ba61bb751 Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 15 Aug 2022 14:10:46 +0300 Subject: [PATCH 34/74] Merge with master --- CMakeLists.txt | 1 - core/blockchain/impl/block_tree_impl.cpp | 14 +- .../consensus/authority/authority_manager.hpp | 17 +- .../authority/impl/authority_manager_impl.cpp | 551 ++++++++++-------- .../authority/impl/authority_manager_impl.hpp | 27 +- .../authority/impl/schedule_node.cpp | 75 +-- .../authority/impl/schedule_node.hpp | 129 +++- core/consensus/grandpa/chain.hpp | 2 +- core/consensus/grandpa/common.hpp | 2 +- core/consensus/grandpa/environment.hpp | 15 +- .../grandpa/impl/environment_impl.cpp | 16 +- .../grandpa/impl/environment_impl.hpp | 16 +- core/consensus/grandpa/impl/grandpa_impl.cpp | 33 +- core/consensus/grandpa/impl/grandpa_impl.hpp | 2 +- .../impl/vote_crypto_provider_impl.cpp | 20 +- .../grandpa/impl/voting_round_impl.cpp | 2 +- .../grandpa/impl/voting_round_impl.hpp | 2 +- core/consensus/grandpa/structs.hpp | 2 +- core/consensus/grandpa/voter_set.cpp | 2 +- core/consensus/grandpa/voter_set.hpp | 6 +- core/consensus/grandpa/voting_round.hpp | 2 +- core/injector/application_injector.cpp | 13 + core/network/peer_manager.hpp | 2 +- core/network/types/grandpa_message.hpp | 13 +- core/primitives/authority.hpp | 51 +- core/primitives/block_data.hpp | 2 +- core/primitives/digest.hpp | 9 + core/primitives/scheduled_change.hpp | 7 + core/runtime/runtime_api/grandpa_api.hpp | 7 + core/runtime/runtime_api/impl/grandpa_api.cpp | 6 + core/runtime/runtime_api/impl/grandpa_api.hpp | 5 +- core/storage/predefined_keys.hpp | 3 - core/storage/rocksdb/rocksdb_util.hpp | 2 + .../docker/kagome-dev/build_and_push.sh | 2 +- .../docker/kagome-dev/minideb.Dockerfile | 2 +- test/core/consensus/authority/CMakeLists.txt | 1 + .../authority/authority_manager_test.cpp | 113 ++-- .../authority/authority_manager_mock.hpp | 3 +- .../consensus/grandpa/voting_round_mock.hpp | 2 +- 39 files changed, 731 insertions(+), 448 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 146a060746..0e694c5fb4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,6 @@ endif() set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(TESTING "Build and run test suite" ON ) - option(CLANG_FORMAT "Enable clang-format target" ON ) option(CLANG_TIDY "Enable clang-tidy checks during compilation" OFF) option(COVERAGE "Enable generation of coverage info" OFF) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 3ef15baf15..ea39d36be6 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1136,20 +1136,20 @@ namespace kagome::blockchain { if (ancestor_node_ptr) { ancestor_depth = ancestor_node_ptr->depth; } else { - auto header_res = header_repo_->getBlockHeader(ancestor); - if (!header_res) { + auto number_res = header_repo_->getNumberByHash(ancestor); + if (!number_res) { return false; } - ancestor_depth = header_res.value().number; + ancestor_depth = number_res.value(); } if (descendant_node_ptr) { descendant_depth = descendant_node_ptr->depth; } else { - auto header_res = header_repo_->getBlockHeader(descendant); - if (!header_res) { + auto number_res = header_repo_->getNumberByHash(descendant); + if (!number_res) { return false; } - descendant_depth = header_res.value().number; + descendant_depth = number_res.value(); } if (descendant_depth < ancestor_depth) { SL_WARN(log_, @@ -1175,6 +1175,7 @@ namespace kagome::blockchain { // else, we need to use a database auto current_hash = descendant; + KAGOME_PROFILE_START(search_finalized_chain) while (current_hash != ancestor) { auto current_header_res = header_repo_->getBlockHeader(current_hash); if (!current_header_res) { @@ -1182,6 +1183,7 @@ namespace kagome::blockchain { } current_hash = current_header_res.value().parent_hash; } + KAGOME_PROFILE_END(search_finalized_chain) return true; } diff --git a/core/consensus/authority/authority_manager.hpp b/core/consensus/authority/authority_manager.hpp index b3c547fd44..00f80b61d6 100644 --- a/core/consensus/authority/authority_manager.hpp +++ b/core/consensus/authority/authority_manager.hpp @@ -10,6 +10,7 @@ #include "common/tagged.hpp" #include "primitives/authority.hpp" +#include "primitives/block_header.hpp" namespace kagome::authority { @@ -19,6 +20,17 @@ namespace kagome::authority { public: virtual ~AuthorityManager() = default; + using HeaderIterator = std::function; + + /** + * Recalculate the authority change graph starting from genesis and up to + * the last finalized block. The result shall be stored in the provided + * storage. This operation may take a considerable amount of time. + * @param header_iter - iterator over finalized block headers + * @return nothing on success, error otherwise + */ + virtual outcome::result recalculateStoredState(primitives::BlockNumber last_finalized_number) = 0; + /** * @return block associated with the root of scheduled changes tree */ @@ -31,7 +43,7 @@ namespace kagome::authority { * finalized * @return outcome authority set */ - virtual std::optional> + virtual std::optional> authorities(const primitives::BlockInfo &block, IsBlockFinalized finalized) const = 0; @@ -59,7 +71,8 @@ namespace kagome::authority { virtual outcome::result applyForcedChange( const primitives::BlockInfo &block, const primitives::AuthorityList &authorities, - primitives::BlockNumber activate_at) = 0; + primitives::BlockNumber delay_start, + size_t delay) = 0; /** * @brief An index of the individual authority in the current authority list diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index deaa4ae1db..af867eef67 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -9,21 +9,22 @@ #include #include +#include #include "application/app_state_manager.hpp" +#include "blockchain/block_header_repository.hpp" #include "blockchain/block_tree.hpp" #include "common/visitor.hpp" #include "consensus/authority/authority_manager_error.hpp" #include "consensus/authority/authority_update_observer_error.hpp" #include "consensus/authority/impl/schedule_node.hpp" -#include "consensus/grandpa/common.hpp" #include "crypto/hasher.hpp" +#include "log/profiling_logger.hpp" #include "runtime/runtime_api/grandpa_api.hpp" -#include "scale/scale.hpp" #include "storage/trie/trie_storage.hpp" using kagome::common::Buffer; -using kagome::consensus::grandpa::MembershipCounter; +using kagome::primitives::AuthoritySetId; namespace kagome::authority { @@ -33,17 +34,23 @@ namespace kagome::authority { std::shared_ptr block_tree, std::shared_ptr trie_storage, std::shared_ptr grandpa_api, - std::shared_ptr hasher) + std::shared_ptr hasher, + std::shared_ptr persistent_storage, + std::shared_ptr header_repo) : config_{std::move(config)}, block_tree_(std::move(block_tree)), trie_storage_(std::move(trie_storage)), grandpa_api_(std::move(grandpa_api)), hasher_(std::move(hasher)), + persistent_storage_{std::move(persistent_storage)}, + header_repo_{std::move(header_repo)}, log_{log::createLogger("AuthorityManager", "authority")} { BOOST_ASSERT(block_tree_ != nullptr); BOOST_ASSERT(grandpa_api_ != nullptr); BOOST_ASSERT(trie_storage_ != nullptr); BOOST_ASSERT(hasher_ != nullptr); + BOOST_ASSERT(persistent_storage_ != nullptr); + BOOST_ASSERT(header_repo_ != nullptr); BOOST_ASSERT(app_state_manager != nullptr); app_state_manager->atPrepare([&] { return prepare(); }); @@ -91,13 +98,13 @@ namespace kagome::authority { return collected; } - outcome::result> fetchSetIdFromTrieStorage( + outcome::result> fetchSetIdFromTrieStorage( storage::trie::TrieStorage const &trie_storage, crypto::Hasher const &hasher, primitives::BlockHeader const &header) { OUTCOME_TRY(batch, trie_storage.getEphemeralBatchAt(header.state_root)); - std::optional set_id_opt; + std::optional set_id_opt; auto current_set_id_keypart = hasher.twox_128(Buffer::fromString("CurrentSetId")); for (auto prefix : {"GrandpaFinality", "Grandpa"}) { @@ -108,13 +115,36 @@ namespace kagome::authority { OUTCOME_TRY(val_opt, batch->tryGet(set_id_key)); if (val_opt.has_value()) { auto &val = val_opt.value(); - set_id_opt.emplace(scale::decode(val.get()).value()); + set_id_opt.emplace(scale::decode(val.get()).value()); break; } } return set_id_opt; } + static const common::Buffer kScheduleGraphRootKey = + common::Buffer::fromString(":authority_manager:schedule_graph_root"); + + outcome::result>> + fetchScheduleGraphRoot(storage::BufferStorage const &storage) { + OUTCOME_TRY(opt_root, storage.tryLoad(kScheduleGraphRootKey)); + if (!opt_root) return std::nullopt; + auto &encoded_root = opt_root.value(); + OUTCOME_TRY(root, scale::decode(encoded_root)); + return std::make_unique(std::move(root)); + } + + outcome::result storeScheduleGraphRoot(storage::BufferStorage &storage, + ScheduleNode const &root) { + OUTCOME_TRY(enc_root, scale::encode(root)); + OUTCOME_TRY(storage.put(kScheduleGraphRootKey, + common::Buffer{std::move(enc_root)})); + return outcome::success(); + } + + static const common::Buffer kForcedSetIdListKey = + common::Buffer::fromString(":authority_manager:forced_set_id_list"); + /** * Collect all consensus messages found in finalized block starting from * {@param finalized_block_hash} and until an authority set change is reached. @@ -229,64 +259,81 @@ namespace kagome::authority { "Error collecting consensus messages from non-finalized blocks: {}", error.message()); - PREPARE_TRY(significant_block, - collectConsensusMsgsUntilNearestSetChangeTo( - collected_msgs, finalized_block, *block_tree_, log_), - "Error collecting consensus messages from finalized blocks: {}", + PREPARE_TRY(opt_root, + fetchScheduleGraphRoot(*persistent_storage_), + "Error fetching authority set from persistent storage: {}", error.message()); + auto last_finalized_block = block_tree_->getLastFinalized(); + if (opt_root + && opt_root.value()->current_block.number + <= last_finalized_block.number) { + root_ = std::move(opt_root.value()); + SL_TRACE(log_, + "Fetched authority set graph root from database with id {}", + root_->current_authorities->id); - primitives::AuthorityList authorities; - { // get voter set id at earliest significant block - const auto &hash = significant_block.hash; - PREPARE_TRY(header, - block_tree_->getBlockHeader(hash), - "Can't get header of block {}: {}", - significant_block, + } else if (last_finalized_block.number == 0) { + auto genesis_hash = block_tree_->getGenesisBlockHash(); + PREPARE_TRY(initial_authorities, + grandpa_api_->authorities(genesis_hash), + "Can't get grandpa authorities for genesis block: {}", error.message()); - + root_ = authority::ScheduleNode::createAsRoot( + std::make_shared( + 0, std::move(initial_authorities)), + {0, genesis_hash}); + } else { + SL_WARN( + log_, + "Storage does not contain valid info about the root authority set; " + "Fall back to obtaining it from the runtime storage (which may " + "fail after a forced authority change happened on chain)"); PREPARE_TRY( - set_id_opt, - fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header), - "Error fetching authority set id from trie storage for block {}: {}", - significant_block, + graph_root_block, + collectConsensusMsgsUntilNearestSetChangeTo( + collected_msgs, finalized_block, *block_tree_, log_), + "Error collecting consensus messages from finalized blocks: {}", error.message()); - - if (not set_id_opt.has_value()) { - log_->critical( - "Can't get grandpa set id for block {}: " - "CurrentSetId not found in Trie storage", - significant_block); - return false; - } - const auto &set_id = set_id_opt.value(); - SL_TRACE(log_, - "Initialized set id from runtime: #{} at block {}", - set_id, - significant_block); - - // Get initial authorities from runtime - PREPARE_TRY(initial_authorities, - grandpa_api_->authorities(hash), + PREPARE_TRY(authorities, + grandpa_api_->authorities(graph_root_block.hash), "Can't get grandpa authorities for block {}: {}", - significant_block, + graph_root_block, error.message()); - authorities = std::move(initial_authorities); - authorities.id = set_id; - } - - auto node = authority::ScheduleNode::createAsRoot(significant_block); - node->actual_authorities = - std::make_shared(std::move(authorities)); + PREPARE_TRY(header, + block_tree_->getBlockHeader(graph_root_block.hash), + "Can't get header of block {}: {}", + graph_root_block, + error.message()); + AuthoritySetId set_id{}; + auto set_id_res = grandpa_api_->current_set_id(graph_root_block.hash); + if (set_id_res) { + set_id = set_id_res.value(); + } else { + PREPARE_TRY( + set_id_, + fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header), + "Failed to fetch authority set id from storage for block {}: {}", + graph_root_block, + error.message()); + set_id = set_id_.value(); + } + auto authority_set = std::make_shared( + set_id, std::move(authorities)); + root_ = authority::ScheduleNode::createAsRoot(authority_set, + graph_root_block); - root_ = std::move(node); + PREPARE_TRY_VOID(storeScheduleGraphRoot(*persistent_storage_, *root_), + "Failed to store schedule graph root: {}", + error.message()); + SL_TRACE(log_, + "Create authority set graph root with id {}, taken from runtime " + "storage", + root_->current_authorities->id); + } while (not collected_msgs.empty()) { const auto &args = collected_msgs.top(); - SL_TRACE(log_, - "Apply consensus message from block {}, engine {}", - args.block, - args.message.consensus_engine_id.toString()); PREPARE_TRY_VOID(onConsensus(args.block, args.message), "Can't apply previous consensus message: {}", error.message()); @@ -299,13 +346,13 @@ namespace kagome::authority { SL_DEBUG(log_, "Actual grandpa authority set (id={}):", - root_->actual_authorities->id); + root_->current_authorities->id); size_t index = 0; - for (const auto &authority : *root_->actual_authorities) { - SL_DEBUG(log_, + for (const auto &authority : *root_->current_authorities) { + SL_TRACE(log_, "{}/{}: id={} weight={}", ++index, - root_->actual_authorities->size(), + root_->current_authorities->authorities.size(), authority.id.id, authority.weight); } @@ -316,15 +363,52 @@ namespace kagome::authority { #undef PREPARE_TRY_VOID #undef PREPARE_TRY + outcome::result AuthorityManagerImpl::recalculateStoredState( + primitives::BlockNumber last_finalized_number) { + auto genesis_hash = block_tree_->getGenesisBlockHash(); + + OUTCOME_TRY(initial_authorities, grandpa_api_->authorities(genesis_hash)); + + root_ = ScheduleNode::createAsRoot( + std::make_shared(0, initial_authorities), + {0, genesis_hash}); + + for (primitives::BlockNumber number = 0; number <= last_finalized_number; + number++) { + auto start = std::chrono::steady_clock::now(); + OUTCOME_TRY(hash, header_repo_->getHashByNumber(number)); + OUTCOME_TRY(header, header_repo_->getBlockHeader(number)); + primitives::BlockInfo info{number, hash}; + + bool has_authority_change = false; + for (auto &msg : header.digest) { + if (auto consensus_msg = boost::get(&msg); + consensus_msg != nullptr) { + OUTCOME_TRY(onConsensus(info, *consensus_msg)); + has_authority_change = true; + } + } + if (has_authority_change) prune(info); + auto end = std::chrono::steady_clock::now(); + SL_TRACE( + log_, + "Process block #{} in {} ms", + number, + std::chrono::duration_cast(end - start) + .count()); + } + return outcome::success(); + } + primitives::BlockInfo AuthorityManagerImpl::base() const { if (not root_) { log_->critical("Authority manager has null root"); std::terminate(); } - return root_->block; + return root_->current_block; } - std::optional> + std::optional> AuthorityManagerImpl::authorities(const primitives::BlockInfo &target_block, IsBlockFinalized finalized) const { auto node = getAppropriateAncestor(target_block); @@ -334,23 +418,33 @@ namespace kagome::authority { } IsBlockFinalized node_in_finalized_chain = - node->block == target_block + node->current_block == target_block ? (bool)finalized - : (node->block == block_tree_->getLastFinalized() - or directChainExists(node->block, - block_tree_->getLastFinalized())); + : node->current_block.number + <= block_tree_->getLastFinalized().number; auto adjusted_node = node->makeDescendant(target_block, node_in_finalized_chain); if (adjusted_node->enabled) { // Original authorities - return adjusted_node->actual_authorities; + SL_DEBUG(log_, + "Pick authority set with id {} for block {}", + adjusted_node->current_authorities->id, + target_block); + auto header = block_tree_->getBlockHeader(target_block.hash).value(); + auto id_from_storage = + fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header) + .value() + .value(); + SL_DEBUG( + log_, "Pick authority set id from trie storage: {}", id_from_storage); + return adjusted_node->current_authorities; } // Zero-weighted authorities - auto authorities = std::make_shared( - *adjusted_node->actual_authorities); + auto authorities = std::make_shared( + *adjusted_node->current_authorities); std::for_each(authorities->begin(), authorities->end(), [](auto &authority) { authority.weight = 0; }); @@ -365,146 +459,151 @@ namespace kagome::authority { "Applying scheduled change on block {} to activate at block {}", block, activate_at); - auto node = getAppropriateAncestor(block); + KAGOME_PROFILE_START(get_appropriate_ancestor) + auto ancestor_node = getAppropriateAncestor(block); + KAGOME_PROFILE_END(get_appropriate_ancestor) - if (not node) { + if (not ancestor_node) { return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; } SL_DEBUG(log_, - "Actual authorities for block {} found on block {} with set id {}", + "Authorities for block {} found on block {} with set id {}", block, - node->block, - node->actual_authorities->id); + ancestor_node->current_block, + ancestor_node->current_authorities->id); auto schedule_change = [&](const std::shared_ptr &node) -> outcome::result { - auto res = node->ensureReadyToSchedule(); - if (!res) { - SL_DEBUG(log_, - "Node is not ready to schedule scheduled change: {}", - res.error().message()); - return res.as_failure(); - } - - auto new_authorities = - std::make_shared(authorities); - new_authorities->id = node->actual_authorities->id + 1; + auto new_authorities = std::make_shared( + node->current_authorities->id + 1, authorities); - // Schedule change - node->scheduled_authorities = std::move(new_authorities); - node->scheduled_after = activate_at; + node->action = + ScheduleNode::ScheduledChange{activate_at, new_authorities}; SL_VERBOSE(log_, "Change is scheduled after block #{} (set id={})", - node->scheduled_after, - node->scheduled_authorities->id); + activate_at, + new_authorities->id); size_t index = 0; - for (auto &authority : *node->scheduled_authorities) { - SL_VERBOSE(log_, - "New authority ({}/{}): id={} weight={}", - ++index, - node->scheduled_authorities->size(), - authority.id.id, - authority.weight); + for (auto &authority : *new_authorities) { + SL_DEBUG(log_, + "New authority ({}/{}): id={} weight={}", + ++index, + new_authorities->authorities.size(), + authority.id.id, + authority.weight); } return outcome::success(); }; - IsBlockFinalized node_in_finalized_chain = - node->block == block_tree_->getLastFinalized() - or directChainExists(node->block, block_tree_->getLastFinalized()); + KAGOME_PROFILE_START(is_ancestor_node_finalized) + IsBlockFinalized is_ancestor_node_finalized = true; + ancestor_node->current_block.number + <= block_tree_->getLastFinalized().number; + KAGOME_PROFILE_END(is_ancestor_node_finalized) - if (node->block == block) { - node->adjust(node_in_finalized_chain); - OUTCOME_TRY(schedule_change(node)); + if (ancestor_node->current_block == block) { + ancestor_node->adjust(is_ancestor_node_finalized); + + OUTCOME_TRY(schedule_change(ancestor_node)); } else { - auto new_node = node->makeDescendant(block, node_in_finalized_chain); + KAGOME_PROFILE_START(make_descendant) + auto new_node = + ancestor_node->makeDescendant(block, is_ancestor_node_finalized); + KAGOME_PROFILE_END(make_descendant) SL_DEBUG(log_, "Make a schedule node for block {}, with actual set id {}", block, - new_node->actual_authorities->id); + new_node->current_authorities->id); + KAGOME_PROFILE_START(schedule_change) OUTCOME_TRY(schedule_change(new_node)); + KAGOME_PROFILE_END(schedule_change) // Reorganize ancestry - reorganize(node, new_node); + KAGOME_PROFILE_START(reorganize) + reorganize(ancestor_node, new_node); + KAGOME_PROFILE_END(reorganize) } return outcome::success(); } outcome::result AuthorityManagerImpl::applyForcedChange( - const primitives::BlockInfo &block, + const primitives::BlockInfo ¤t_block, const primitives::AuthorityList &authorities, - primitives::BlockNumber activate_at) { + primitives::BlockNumber delay_start, + size_t delay) { SL_DEBUG(log_, - "Applying forced change on block {} to activate at block {}", - block, - activate_at); - auto node = getAppropriateAncestor(block); - - if (not node) { + "Applying forced change (delay start: {}, delay: {}) on block {} " + "to activate at block {}", + delay_start, + delay, + current_block, + delay_start + delay); + OUTCOME_TRY(delay_start_header, + block_tree_->getBlockHeader(delay_start + 1)); + auto ancestor_node = + getAppropriateAncestor({delay_start, delay_start_header.parent_hash}); + + if (not ancestor_node) { return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; } + SL_DEBUG(log_, + "Found previous authority change at block {} with set id {}", + ancestor_node->current_block, + ancestor_node->current_authorities->id); + auto force_change = [&](const std::shared_ptr &node) -> outcome::result { - auto res = node->ensureReadyToSchedule(); - if (!res) { - SL_DEBUG(log_, - "Node is not ready to schedule forced change: {}", - res.error().message()); - return res.as_failure(); - } - - auto new_authorities = - std::make_shared(authorities); - new_authorities->id = node->actual_authorities->id + 1; + auto new_authorities = std::make_shared( + node->current_authorities->id + 1, authorities); // Force changes - if (node->block.number >= activate_at) { - node->actual_authorities = std::move(new_authorities); + if (node->current_block.number >= delay_start + delay) { + node->current_authorities = new_authorities; + SL_VERBOSE(log_, + "Change has been forced on block #{} (set id={})", + delay_start + delay, + node->current_authorities->id); } else { - node->forced_authorities = - std::make_shared(authorities); - node->forced_for = activate_at; + node->action = + ScheduleNode::ForcedChange{delay_start, delay, new_authorities}; + SL_VERBOSE(log_, + "Change will be forced on block #{} (set id={})", + delay_start + delay, + new_authorities->id); } - SL_VERBOSE(log_, - "Change will be forced on block #{} (set id={})", - activate_at, - node->forced_authorities->id); size_t index = 0; - for (auto &authority : *node->forced_authorities) { - SL_VERBOSE(log_, - "New authority ({}/{}): id={} weight={}", - ++index, - node->forced_authorities->size(), - authority.id.id, - authority.weight); + for (auto &authority : *new_authorities) { + SL_DEBUG(log_, + "New authority ({}/{}): id={} weight={}", + ++index, + new_authorities->authorities.size(), + authority.id.id, + authority.weight); } return outcome::success(); }; - IsBlockFinalized node_in_finalized_chain = - node->block == block_tree_->getLastFinalized() - or directChainExists(node->block, block_tree_->getLastFinalized()); + OUTCOME_TRY(delay_start_child, + block_tree_->getBlockHeader(delay_start + 1)); - if (node->block == block) { - node->adjust(node_in_finalized_chain); - OUTCOME_TRY(force_change(node)); - } else { - auto new_node = node->makeDescendant(block, node_in_finalized_chain); + auto new_node = ancestor_node->makeDescendant( + {delay_start, delay_start_child.parent_hash}, true); - OUTCOME_TRY(force_change(new_node)); + OUTCOME_TRY(force_change(new_node)); - // Reorganize ancestry - reorganize(node, new_node); - } + // Reorganize ancestry + ancestor_node->descendants.clear(); + ancestor_node->descendants.push_back(new_node); + new_node->descendants.clear(); // reset all pending scheduled changes return outcome::success(); } @@ -526,33 +625,32 @@ namespace kagome::authority { auto disable_authority = [&](const std::shared_ptr &node) -> outcome::result { // Make changed authorities - auto authorities = std::make_shared( - *node->actual_authorities); + auto new_authority_set = std::make_shared( + *node->current_authorities); // Check if index not out of bound - if (authority_index >= node->actual_authorities->size()) { + if (authority_index >= node->current_authorities->authorities.size()) { return AuthorityUpdateObserverError::WRONG_AUTHORITY_INDEX; } - (*authorities)[authority_index].weight = 0; - node->actual_authorities = std::move(authorities); + new_authority_set->authorities[authority_index].weight = 0; + node->current_authorities = std::move(new_authority_set); SL_VERBOSE( log_, "Authority id={} (index={} in set id={}) is disabled on block #{}", - (*node->actual_authorities)[authority_index].id.id, + node->current_authorities->authorities[authority_index].id.id, authority_index, - node->actual_authorities->id, - node->block.number); + node->current_authorities->id, + node->current_block.number); return outcome::success(); }; IsBlockFinalized node_in_finalized_chain = - node->block == block_tree_->getLastFinalized() - or directChainExists(node->block, block_tree_->getLastFinalized()); + node->current_block.number <= block_tree_->getLastFinalized().number; - if (node->block == block) { + if (node->current_block == block) { node->adjust(node_in_finalized_chain); OUTCOME_TRY(disable_authority(node)); } else { @@ -563,10 +661,10 @@ namespace kagome::authority { // Reorganize ancestry auto descendants = std::move(node->descendants); for (auto &descendant : descendants) { - if (directChainExists(block, descendant->block)) { + if (directChainExists(block, descendant->current_block)) { // Propagate change to descendants - if (descendant->actual_authorities == node->actual_authorities) { - descendant->actual_authorities = new_node->actual_authorities; + if (descendant->current_authorities == node->current_authorities) { + descendant->current_authorities = new_node->current_authorities; } new_node->descendants.emplace_back(std::move(descendant)); } else { @@ -590,22 +688,21 @@ namespace kagome::authority { } IsBlockFinalized node_in_finalized_chain = - node->block == block_tree_->getLastFinalized() - or directChainExists(node->block, block_tree_->getLastFinalized()); + node->current_block.number <= block_tree_->getLastFinalized().number; auto new_node = node->makeDescendant(block, node_in_finalized_chain); - OUTCOME_TRY(new_node->ensureReadyToSchedule()); - - new_node->pause_after = activate_at; + new_node->action = ScheduleNode::Pause{activate_at}; - SL_VERBOSE(log_, "Scheduled pause after block #{}", new_node->block.number); + SL_VERBOSE(log_, + "Scheduled pause after block #{}", + new_node->current_block.number); // Reorganize ancestry auto descendants = std::move(node->descendants); for (auto &descendant : descendants) { auto &ancestor = - directChainExists(block, descendant->block) ? new_node : node; + block.number <= descendant->current_block.number ? new_node : node; ancestor->descendants.emplace_back(std::move(descendant)); } node->descendants.emplace_back(std::move(new_node)); @@ -622,17 +719,15 @@ namespace kagome::authority { } IsBlockFinalized node_in_finalized_chain = - node->block == block_tree_->getLastFinalized() - or directChainExists(node->block, block_tree_->getLastFinalized()); + node->current_block.number <= block_tree_->getLastFinalized().number; auto new_node = node->makeDescendant(block, node_in_finalized_chain); - OUTCOME_TRY(new_node->ensureReadyToSchedule()); + new_node->action = ScheduleNode::Resume{activate_at}; - new_node->resume_for = activate_at; - - SL_VERBOSE( - log_, "Resuming will be done at block #{}", new_node->block.number); + SL_VERBOSE(log_, + "Resuming will be done at block #{}", + new_node->current_block.number); // Reorganize ancestry reorganize(node, new_node); @@ -643,28 +738,12 @@ namespace kagome::authority { outcome::result AuthorityManagerImpl::onConsensus( const primitives::BlockInfo &block, const primitives::Consensus &message) { - if (message.consensus_engine_id == primitives::kBabeEngineId) { - OUTCOME_TRY(decoded, message.decode()); - // TODO(xDimon): Perhaps it needs to be refactored. - // It is better handle babe digests here - // Issue: https://github.com/soramitsu/kagome/issues/740 - return visit_in_place( - decoded.asBabeDigest(), - [](const primitives::NextEpochData &msg) -> outcome::result { - return outcome::success(); - }, - [](const primitives::OnDisabled &msg) { - // Note: This event type won't be used anymore and must be ignored - return outcome::success(); - }, - [](const primitives::NextConfigData &msg) { - return outcome::success(); - }, - [](auto &) { - return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; - }); - } if (message.consensus_engine_id == primitives::kGrandpaEngineId) { + SL_TRACE(log_, + "Apply consensus message from block {}, engine {}", + block, + message.consensus_engine_id.toString()); + OUTCOME_TRY(decoded, message.decode()); return visit_in_place( decoded.asGrandpaDigest(), @@ -675,7 +754,7 @@ namespace kagome::authority { }, [this, &block](const primitives::ForcedChange &msg) { return applyForcedChange( - block, msg.authorities, block.number + msg.subchain_length); + block, msg.authorities, msg.delay_start, msg.subchain_length); }, [this, &block](const primitives::OnDisabled &msg) { return applyOnDisabled(block, msg.authority_index); @@ -689,21 +768,25 @@ namespace kagome::authority { [](auto &) { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; }); + } else if (message.consensus_engine_id == primitives::kBabeEngineId) { + // ignore + return outcome::success(); + } else { - SL_WARN(log_, - "Unknown consensus engine id in block {}: {}", - block, - message.consensus_engine_id.toString()); + SL_DEBUG(log_, + "Unknown consensus engine id in block {}: {}", + block, + message.consensus_engine_id.toString()); return outcome::success(); } } void AuthorityManagerImpl::prune(const primitives::BlockInfo &block) { - if (block == root_->block) { + if (block == root_->current_block) { return; } - if (block.number < root_->block.number) { + if (block.number < root_->current_block.number) { return; } @@ -713,7 +796,7 @@ namespace kagome::authority { return; } - if (node->block == block) { + if (node->current_block == block) { // Rebase root_ = std::move(node); @@ -722,13 +805,14 @@ namespace kagome::authority { auto new_node = node->makeDescendant(block, true); auto descendants = std::move(node->descendants); for (auto &descendant : descendants) { - if (directChainExists(block, descendant->block)) { + if (directChainExists(block, descendant->current_block)) { new_node->descendants.emplace_back(std::move(descendant)); } } root_ = std::move(new_node); } + storeScheduleGraphRoot(*persistent_storage_, *root_).value(); SL_VERBOSE(log_, "Prune authority manager upto block {}", block); } @@ -736,18 +820,19 @@ namespace kagome::authority { std::shared_ptr AuthorityManagerImpl::getAppropriateAncestor( const primitives::BlockInfo &block) const { BOOST_ASSERT(root_ != nullptr); - std::shared_ptr ancestor; + // Target block is not descendant of the current root - if (root_->block.number > block.number - || (root_->block != block - && not directChainExists(root_->block, block))) { - return ancestor; + if (root_->current_block.number > block.number + || (root_->current_block != block + && not directChainExists(root_->current_block, block))) { + return nullptr; } - ancestor = root_; - while (ancestor->block != block) { + std::shared_ptr ancestor = root_; + while (ancestor->current_block != block) { bool goto_next_generation = false; for (const auto &node : ancestor->descendants) { - if (node->block == block || directChainExists(node->block, block)) { + if (node->current_block == block + || directChainExists(node->current_block, block)) { ancestor = node; goto_next_generation = true; break; @@ -763,6 +848,11 @@ namespace kagome::authority { bool AuthorityManagerImpl::directChainExists( const primitives::BlockInfo &ancestor, const primitives::BlockInfo &descendant) const { + SL_TRACE(log_, + "Looking if direct chain exists between {} and {}", + ancestor, + descendant); + KAGOME_PROFILE_START(direct_chain_exists) // Any block is descendant of genesis if (ancestor.number <= 1 && ancestor.number < descendant.number) { return true; @@ -770,6 +860,7 @@ namespace kagome::authority { auto result = ancestor.number < descendant.number && block_tree_->hasDirectChain(ancestor.hash, descendant.hash); + KAGOME_PROFILE_END(direct_chain_exists) return result; } @@ -778,22 +869,24 @@ namespace kagome::authority { std::shared_ptr new_node) { auto descendants = std::move(node->descendants); for (auto &descendant : descendants) { - auto &ancestor = directChainExists(new_node->block, descendant->block) - ? new_node - : node; + auto &ancestor = + new_node->current_block.number < descendant->current_block.number + ? new_node + : node; // Apply if delay will be passed for descendant - if (ancestor->forced_for != ScheduleNode::INACTIVE) { - if (descendant->block.number >= ancestor->forced_for) { - descendant->actual_authorities = ancestor->forced_authorities; - descendant->forced_authorities.reset(); - descendant->forced_for = ScheduleNode::INACTIVE; + if (auto *forced_change = + boost::get(&ancestor->action)) { + if (descendant->current_block.number + >= forced_change->delay_start + forced_change->delay_length) { + descendant->current_authorities = forced_change->new_authorities; + descendant->action = ScheduleNode::NoAction{}; } } - if (ancestor->resume_for != ScheduleNode::INACTIVE) { - if (descendant->block.number >= ancestor->resume_for) { + if (auto *resume = boost::get(&ancestor->action)) { + if (descendant->current_block.number >= resume->applied_block) { descendant->enabled = true; - descendant->resume_for = ScheduleNode::INACTIVE; + descendant->action = ScheduleNode::NoAction{}; } } @@ -818,14 +911,14 @@ namespace kagome::authority { return; } - if (ancestor->block == block) { + if (ancestor->current_block == block) { ancestor = std::const_pointer_cast(ancestor->parent.lock()); } auto it = std::find_if(ancestor->descendants.begin(), ancestor->descendants.end(), [&block](std::shared_ptr node) { - return node->block == block; + return node->current_block == block; }); if (it != ancestor->descendants.end()) { diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 44c6c59a35..51e9caf4f9 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -11,6 +11,9 @@ #include "crypto/hasher.hpp" #include "log/logger.hpp" +#include "primitives/authority.hpp" +#include "primitives/block_header.hpp" +#include "storage/buffer_map_types.hpp" namespace kagome::application { class AppStateManager; @@ -20,14 +23,17 @@ namespace kagome::authority { } namespace kagome::blockchain { class BlockTree; -} + class BlockHeaderRepository; +} // namespace kagome::blockchain namespace kagome::primitives { - struct AuthorityList; struct BabeConfiguration; } // namespace kagome::primitives + namespace kagome::runtime { class GrandpaApi; -} + class Executor; +} // namespace kagome::runtime + namespace kagome::storage::trie { class TrieStorage; } @@ -53,16 +59,20 @@ namespace kagome::authority { std::shared_ptr block_tree, std::shared_ptr trie_storage, std::shared_ptr grandpa_api, - std::shared_ptr hash); + std::shared_ptr hash, + std::shared_ptr persistent_storage, + std::shared_ptr header_repo); ~AuthorityManagerImpl() override = default; - // Prepare for work + outcome::result recalculateStoredState( + primitives::BlockNumber last_finalized_number) override; + bool prepare(); primitives::BlockInfo base() const override; - std::optional> authorities( + std::optional> authorities( const primitives::BlockInfo &target_block, IsBlockFinalized finalized) const override; @@ -74,7 +84,8 @@ namespace kagome::authority { outcome::result applyForcedChange( const primitives::BlockInfo &block, const primitives::AuthorityList &authorities, - primitives::BlockNumber activate_at) override; + primitives::BlockNumber delay_start, + size_t delay) override; outcome::result applyOnDisabled(const primitives::BlockInfo &block, uint64_t authority_index) override; @@ -121,6 +132,8 @@ namespace kagome::authority { std::shared_ptr trie_storage_; std::shared_ptr grandpa_api_; std::shared_ptr hasher_; + std::shared_ptr persistent_storage_; + std::shared_ptr header_repo_; std::shared_ptr root_; log::Logger log_; diff --git a/core/consensus/authority/impl/schedule_node.cpp b/core/consensus/authority/impl/schedule_node.cpp index d221f6008d..b9eae28cd6 100644 --- a/core/consensus/authority/impl/schedule_node.cpp +++ b/core/consensus/authority/impl/schedule_node.cpp @@ -11,59 +11,43 @@ namespace kagome::authority { ScheduleNode::ScheduleNode( const std::shared_ptr &ancestor, primitives::BlockInfo block) - : block(block), parent(ancestor) { - BOOST_ASSERT((bool)ancestor); + : current_block(block), parent(ancestor) { + BOOST_ASSERT(ancestor != nullptr); } std::shared_ptr ScheduleNode::createAsRoot( + std::shared_ptr current_authorities, primitives::BlockInfo block) { auto fake_parent = std::make_shared(ScheduleNode()); - return std::make_shared(fake_parent, block); - } - - outcome::result ScheduleNode::ensureReadyToSchedule() const { - if (scheduled_after != INACTIVE) { - return AuthorityUpdateObserverError::NO_SCHEDULED_CHANGE_APPLIED_YET; - } - if (forced_for != INACTIVE) { - return AuthorityUpdateObserverError::NO_FORCED_CHANGE_APPLIED_YET; - } - if (pause_after != INACTIVE) { - return AuthorityUpdateObserverError::NO_PAUSE_APPLIED_YET; - } - if (resume_for != INACTIVE) { - return AuthorityUpdateObserverError::NO_RESUME_APPLIED_YET; - } - return outcome::success(); + auto node = std::make_shared(fake_parent, block); + node->current_authorities = current_authorities; + return node; } void ScheduleNode::adjust(IsBlockFinalized finalized) { - // Has ScheduledChange - if (scheduled_after != INACTIVE) { - if (finalized && scheduled_after <= block.number) { - actual_authorities = std::move(scheduled_authorities); - scheduled_after = INACTIVE; - } - } - // Has ForcedChange - else if (forced_for != INACTIVE) { - if (forced_for <= block.number) { - actual_authorities = std::move(forced_authorities); - forced_for = INACTIVE; + if (auto scheduled_change = boost::get(&action); + finalized && scheduled_change != nullptr) { + if (scheduled_change->applied_block <= current_block.number) { + current_authorities = std::move(scheduled_change->new_authorities); + action = NoAction{}; } - } - // Has planned pause - else if (pause_after != INACTIVE) { - if (finalized && pause_after <= block.number) { + } else if (auto pause = boost::get(&action); + finalized && pause != nullptr) { + if (pause->applied_block <= current_block.number) { enabled = false; - pause_after = INACTIVE; + action = NoAction{}; } - } - // Has planned resume - else if (resume_for != INACTIVE) { - if (resume_for <= block.number) { + } else if (auto forced_change = boost::get(&action); + forced_change != nullptr) { + if (forced_change->delay_start + forced_change->delay_length + <= current_block.number) { + current_authorities = std::move(forced_change->new_authorities); + action = NoAction{}; + } + } else if (auto resume = boost::get(&action); resume != nullptr) { + if (resume->applied_block <= current_block.number) { enabled = true; - resume_for = INACTIVE; + action = NoAction{}; } } } @@ -73,14 +57,9 @@ namespace kagome::authority { IsBlockFinalized finalized) const { auto node = std::make_shared(shared_from_this(), target_block); - node->actual_authorities = actual_authorities; + node->current_authorities = current_authorities; node->enabled = enabled; - node->forced_for = forced_for; - node->forced_authorities = forced_authorities; - node->scheduled_after = scheduled_after; - node->scheduled_authorities = scheduled_authorities; - node->pause_after = pause_after; - node->resume_for = resume_for; + node->action = action; node->adjust(finalized); return node; } diff --git a/core/consensus/authority/impl/schedule_node.hpp b/core/consensus/authority/impl/schedule_node.hpp index 79fd29233a..c5640bb866 100644 --- a/core/consensus/authority/impl/schedule_node.hpp +++ b/core/consensus/authority/impl/schedule_node.hpp @@ -6,6 +6,8 @@ #ifndef KAGOME_CONSENSUS_AUTHORITIES_SCHEDULE_NODE #define KAGOME_CONSENSUS_AUTHORITIES_SCHEDULE_NODE +#include + #include "common/tagged.hpp" #include "primitives/authority.hpp" @@ -20,8 +22,6 @@ namespace kagome::authority { */ class ScheduleNode : public std::enable_shared_from_this { public: - static constexpr primitives::BlockNumber INACTIVE = 0; - ScheduleNode() = default; ScheduleNode(const std::shared_ptr &ancestor, @@ -30,15 +30,9 @@ namespace kagome::authority { /// Creates schedule node as root /// @result schedule node static std::shared_ptr createAsRoot( + std::shared_ptr current_authorities, primitives::BlockInfo block); - /// Ensure if all significant changes is applied and node is ready to - /// schedule next change - /// @result success or error - outcome::result ensureReadyToSchedule() const; - - /// Changes actual authorities as if corresponding block is finalized or not - /// @param finalized - true if block is finalized void adjust(IsBlockFinalized finalized); /// Creates descendant schedule node for block @@ -48,27 +42,108 @@ namespace kagome::authority { std::shared_ptr makeDescendant( const primitives::BlockInfo &block, IsBlockFinalized finalized) const; - const primitives::BlockInfo block{}; + struct NoAction { + friend inline ::scale::ScaleEncoderStream &operator<<( + ::scale::ScaleEncoderStream &s, const NoAction &change) { + return s; + } + + friend inline ::scale::ScaleDecoderStream &operator>>( + ::scale::ScaleDecoderStream &s, NoAction &change) { + return s; + } + }; + + struct ScheduledChange { + primitives::BlockNumber applied_block{}; + std::shared_ptr new_authorities{}; + + friend inline ::scale::ScaleEncoderStream &operator<<( + ::scale::ScaleEncoderStream &s, const ScheduledChange &change) { + return s << change.applied_block << *change.new_authorities; + } + + friend inline ::scale::ScaleDecoderStream &operator>>( + ::scale::ScaleDecoderStream &s, ScheduledChange &change) { + auto authority_list = std::make_shared(); + s >> change.applied_block >> *authority_list; + change.new_authorities = std::move(authority_list); + return s; + } + }; + + struct ForcedChange { + primitives::BlockNumber delay_start{}; + size_t delay_length{}; + std::shared_ptr new_authorities{}; + + friend inline ::scale::ScaleEncoderStream &operator<<( + ::scale::ScaleEncoderStream &s, const ForcedChange &change) { + return s << change.delay_start << change.delay_length + << *change.new_authorities; + } + + friend inline ::scale::ScaleDecoderStream &operator>>( + ::scale::ScaleDecoderStream &s, ForcedChange &change) { + auto authority_list = std::make_shared(); + s >> change.delay_start >> change.delay_length >> *authority_list; + change.new_authorities = std::move(authority_list); + return s; + } + }; + + struct Pause { + primitives::BlockNumber applied_block{}; + + friend inline ::scale::ScaleEncoderStream &operator<<( + ::scale::ScaleEncoderStream &s, const Pause &change) { + return s << change.applied_block; + } + + friend inline ::scale::ScaleDecoderStream &operator>>( + ::scale::ScaleDecoderStream &s, Pause &change) { + return s >> change.applied_block; + } + }; + + struct Resume { + primitives::BlockNumber applied_block{}; + + friend inline ::scale::ScaleEncoderStream &operator<<( + ::scale::ScaleEncoderStream &s, const Resume &change) { + return s << change.applied_block; + } + + friend inline ::scale::ScaleDecoderStream &operator>>( + ::scale::ScaleDecoderStream &s, Resume &change) { + return s >> change.applied_block; + } + }; + + friend inline ::scale::ScaleEncoderStream &operator<<( + ::scale::ScaleEncoderStream &s, const ScheduleNode &node) { + return s << node.enabled << node.current_block + << *node.current_authorities << node.action; + } + + friend inline ::scale::ScaleDecoderStream &operator>>( + ::scale::ScaleDecoderStream &s, ScheduleNode &node) { + auto current_authority_list = + std::make_shared(); + s >> node.enabled + >> const_cast(node.current_block) + >> *current_authority_list >> node.action; + node.current_authorities = std::move(current_authority_list); + return s; + } + + const primitives::BlockInfo current_block{}; std::weak_ptr parent; std::vector> descendants{}; - - // Current authorities - std::shared_ptr actual_authorities; + boost::variant + action; + std::shared_ptr current_authorities; bool enabled = true; - - // For scheduled changes - primitives::BlockNumber scheduled_after = INACTIVE; - std::shared_ptr scheduled_authorities{}; - - // For forced changed - primitives::BlockNumber forced_for = INACTIVE; - std::shared_ptr forced_authorities{}; - - // For pause - primitives::BlockNumber pause_after = INACTIVE; - - // For resume - primitives::BlockNumber resume_for = INACTIVE; }; } // namespace kagome::authority diff --git a/core/consensus/grandpa/chain.hpp b/core/consensus/grandpa/chain.hpp index 2c23832545..b213f38f26 100644 --- a/core/consensus/grandpa/chain.hpp +++ b/core/consensus/grandpa/chain.hpp @@ -53,7 +53,7 @@ namespace kagome::consensus::grandpa { */ virtual outcome::result bestChainContaining( const primitives::BlockHash &base, - std::optional voter_set_id) const = 0; + std::optional voter_set_id) const = 0; /** * @returns true if {@param block} is a descendant of or equal to the diff --git a/core/consensus/grandpa/common.hpp b/core/consensus/grandpa/common.hpp index e6b2844bbc..5e63283257 100644 --- a/core/consensus/grandpa/common.hpp +++ b/core/consensus/grandpa/common.hpp @@ -23,7 +23,7 @@ namespace kagome::consensus::grandpa { using BlockHash = primitives::BlockHash; using BlockNumber = primitives::BlockNumber; using RoundNumber = uint64_t; - using MembershipCounter = uint64_t; + using VoterSetId = uint64_t; using Clock = clock::SteadyClock; using Duration = Clock::Duration; diff --git a/core/consensus/grandpa/environment.hpp b/core/consensus/grandpa/environment.hpp index 585b7b029d..27d7aa6215 100644 --- a/core/consensus/grandpa/environment.hpp +++ b/core/consensus/grandpa/environment.hpp @@ -56,7 +56,7 @@ namespace kagome::consensus::grandpa { */ virtual outcome::result onCatchUpRequested( const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number) = 0; /** @@ -64,7 +64,7 @@ namespace kagome::consensus::grandpa { */ virtual outcome::result onCatchUpRespond( const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number, std::vector prevote_justification, std::vector precommit_justification, @@ -75,7 +75,7 @@ namespace kagome::consensus::grandpa { */ virtual void sendState(const libp2p::peer::PeerId &peer_id, const MovableRoundState &state, - MembershipCounter voter_set_id) = 0; + VoterSetId voter_set_id) = 0; /** * Note that we've done a vote in the given round. @@ -83,7 +83,7 @@ namespace kagome::consensus::grandpa { * provided set_id and given vote is ready to be sent. */ virtual outcome::result onVoted(RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, const SignedMessage &vote) = 0; /** @@ -93,7 +93,7 @@ namespace kagome::consensus::grandpa { */ virtual outcome::result onCommitted( RoundNumber round, - MembershipCounter voter_ser_id, + VoterSetId voter_ser_id, const BlockInfo &vote, const GrandpaJustification &justification) = 0; @@ -104,8 +104,7 @@ namespace kagome::consensus::grandpa { * @param last_finalized last known finalized block */ virtual outcome::result onNeighborMessageSent( - RoundNumber round, - MembershipCounter set_id, + RoundNumber round, VoterSetId set_id, BlockNumber last_finalized) = 0; /** @@ -125,7 +124,7 @@ namespace kagome::consensus::grandpa { * should be applied to the storage */ virtual outcome::result finalize( - MembershipCounter id, const GrandpaJustification &justification) = 0; + VoterSetId id, const GrandpaJustification &justification) = 0; /** * Returns justification for given block diff --git a/core/consensus/grandpa/impl/environment_impl.cpp b/core/consensus/grandpa/impl/environment_impl.cpp index e50eaa28ff..2a50e7fd63 100644 --- a/core/consensus/grandpa/impl/environment_impl.cpp +++ b/core/consensus/grandpa/impl/environment_impl.cpp @@ -62,7 +62,7 @@ namespace kagome::consensus::grandpa { outcome::result EnvironmentImpl::bestChainContaining( const BlockHash &base, - std::optional voter_set_id) const { + std::optional voter_set_id) const { SL_DEBUG(logger_, "Finding best chain containing block {}", base); OUTCOME_TRY(best_block, block_tree_->getBestContaining(base, std::nullopt)); @@ -92,7 +92,7 @@ namespace kagome::consensus::grandpa { outcome::result EnvironmentImpl::onCatchUpRequested( const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number) { SL_DEBUG( logger_, "Send Catch-Up-Request beginning with round {}", round_number); @@ -104,7 +104,7 @@ namespace kagome::consensus::grandpa { outcome::result EnvironmentImpl::onCatchUpRespond( const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number, std::vector prevote_justification, std::vector precommit_justification, @@ -121,7 +121,7 @@ namespace kagome::consensus::grandpa { } outcome::result EnvironmentImpl::onVoted(RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, const SignedMessage &vote) { SL_DEBUG(logger_, "Round #{}: Send {} signed by {} for block {}", @@ -142,7 +142,7 @@ namespace kagome::consensus::grandpa { void EnvironmentImpl::sendState(const libp2p::peer::PeerId &peer_id, const MovableRoundState &state, - MembershipCounter voter_set_id) { + VoterSetId voter_set_id) { auto send = [&](const SignedMessage &vote) { SL_DEBUG(logger_, "Round #{}: Send {} signed by {} for block {} (as send state)", @@ -174,7 +174,7 @@ namespace kagome::consensus::grandpa { outcome::result EnvironmentImpl::onCommitted( RoundNumber round, - MembershipCounter voter_ser_id, + VoterSetId voter_ser_id, const BlockInfo &vote, const GrandpaJustification &justification) { if (round == 0) { @@ -199,7 +199,7 @@ namespace kagome::consensus::grandpa { } outcome::result EnvironmentImpl::onNeighborMessageSent( - RoundNumber round, MembershipCounter set_id, BlockNumber last_finalized) { + RoundNumber round, VoterSetId set_id, BlockNumber last_finalized) { SL_DEBUG(logger_, "Round #{}: Send neighbor message", round); network::GrandpaNeighborMessage message{.round_number = round, @@ -232,7 +232,7 @@ namespace kagome::consensus::grandpa { } outcome::result EnvironmentImpl::finalize( - MembershipCounter id, const GrandpaJustification &grandpa_justification) { + VoterSetId id, const GrandpaJustification &grandpa_justification) { primitives::Justification justification; OUTCOME_TRY(enc, scale::encode(grandpa_justification)); justification.data.put(enc); diff --git a/core/consensus/grandpa/impl/environment_impl.hpp b/core/consensus/grandpa/impl/environment_impl.hpp index 1b50cd8bf9..9281451c04 100644 --- a/core/consensus/grandpa/impl/environment_impl.hpp +++ b/core/consensus/grandpa/impl/environment_impl.hpp @@ -49,18 +49,18 @@ namespace kagome::consensus::grandpa { outcome::result bestChainContaining( const primitives::BlockHash &base, - std::optional voter_set_id) const override; + std::optional voter_set_id) const override; // Environment methods outcome::result onCatchUpRequested( const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number) override; outcome::result onCatchUpRespond( const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number, std::vector prevote_justification, std::vector precommit_justification, @@ -68,21 +68,21 @@ namespace kagome::consensus::grandpa { void sendState(const libp2p::peer::PeerId &peer_id, const MovableRoundState &state, - MembershipCounter voter_set_id) override; + VoterSetId voter_set_id) override; outcome::result onVoted(RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, const SignedMessage &vote) override; outcome::result onCommitted( RoundNumber round, - MembershipCounter voter_ser_id, + VoterSetId voter_ser_id, const BlockInfo &vote, const GrandpaJustification &justification) override; outcome::result onNeighborMessageSent( RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, BlockNumber last_finalized) override; outcome::result applyJustification( @@ -90,7 +90,7 @@ namespace kagome::consensus::grandpa { const primitives::Justification &justification) override; outcome::result finalize( - MembershipCounter id, + VoterSetId id, const GrandpaJustification &justification) override; // Getters diff --git a/core/consensus/grandpa/impl/grandpa_impl.cpp b/core/consensus/grandpa/impl/grandpa_impl.cpp index 6efdf997b4..8de9e686f2 100644 --- a/core/consensus/grandpa/impl/grandpa_impl.cpp +++ b/core/consensus/grandpa/impl/grandpa_impl.cpp @@ -98,10 +98,10 @@ namespace kagome::consensus::grandpa { round_state.last_finalized_block); return false; } - auto &authorities = authorities_res.value(); + auto &authority_set = authorities_res.value(); - auto voters = std::make_shared(authorities->id); - for (const auto &authority : *authorities) { + auto voters = std::make_shared(authority_set->id); + for (const auto &authority : authority_set->authorities) { auto res = voters->insert(primitives::GrandpaSessionKey(authority.id.id), authority.weight); if (res.has_error()) { @@ -176,11 +176,11 @@ namespace kagome::consensus::grandpa { std::abort(); } - auto &authorities = authorities_opt.value(); - BOOST_ASSERT(not authorities->empty()); + auto &authority_set = authorities_opt.value(); + BOOST_ASSERT(not authority_set->authorities.empty()); - auto voters = std::make_shared(authorities->id); - for (const auto &authority : *authorities) { + auto voters = std::make_shared(authority_set->id); + for (const auto &authority : authority_set->authorities) { auto res = voters->insert(primitives::GrandpaSessionKey(authority.id.id), authority.weight); if (res.has_error()) { @@ -221,7 +221,7 @@ namespace kagome::consensus::grandpa { } std::optional> GrandpaImpl::selectRound( - RoundNumber round_number, std::optional voter_set_id) { + RoundNumber round_number, std::optional voter_set_id) { std::shared_ptr round = current_round_; while (round != nullptr) { @@ -433,7 +433,6 @@ namespace kagome::consensus::grandpa { msg.round_number, peer_id); throw std::runtime_error("Need not ensure if it is correct"); - return; } SL_DEBUG(logger_, @@ -501,10 +500,10 @@ namespace kagome::consensus::grandpa { round_state.finalized.value()); return; } - auto &authorities = authorities_opt.value(); + auto &authority_set = authorities_opt.value(); auto voters = std::make_shared(msg.voter_set_id); - for (const auto &authority : *authorities) { + for (const auto &authority : authority_set->authorities) { auto res = voters->insert( primitives::GrandpaSessionKey(authority.id.id), authority.weight); if (res.has_error()) { @@ -847,22 +846,22 @@ namespace kagome::consensus::grandpa { block_info); return VotingRoundError::NO_KNOWN_AUTHORITIES_FOR_BLOCK; } - auto &authorities = authorities_opt.value(); + auto &authority_set = authorities_opt.value(); // This is justification for non-actual round - if (authorities->id < current_round_->voterSetId() - or (authorities->id == current_round_->voterSetId() + if (authority_set->id < current_round_->voterSetId() + or (authority_set->id == current_round_->voterSetId() && justification.round_number < current_round_->roundNumber())) { return VotingRoundError::JUSTIFICATION_FOR_ROUND_IN_PAST; } - if (authorities->id > current_round_->voterSetId() + 1) { + if (authority_set->id > current_round_->voterSetId() + 1) { return VotingRoundError::WRONG_ORDER_OF_VOTER_SET_ID; } - auto voters = std::make_shared(authorities->id); - for (const auto &authority : *authorities) { + auto voters = std::make_shared(authority_set->id); + for (const auto &authority : authority_set->authorities) { auto res = voters->insert( primitives::GrandpaSessionKey(authority.id.id), authority.weight); if (res.has_error()) { diff --git a/core/consensus/grandpa/impl/grandpa_impl.hpp b/core/consensus/grandpa/impl/grandpa_impl.hpp index 8b594f15a1..2d43005e8e 100644 --- a/core/consensus/grandpa/impl/grandpa_impl.hpp +++ b/core/consensus/grandpa/impl/grandpa_impl.hpp @@ -219,7 +219,7 @@ namespace kagome::consensus::grandpa { */ std::optional> selectRound( RoundNumber round_number, - std::optional voter_set_id); + std::optional voter_set_id); /** * @return Get grandpa::MovableRoundState for the last completed round diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index add646bc06..12441021e2 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -33,11 +33,21 @@ namespace kagome::consensus::grandpa { bool VoteCryptoProviderImpl::verify(const SignedMessage &vote, RoundNumber number) const { - auto payload = - scale::encode(vote.message, round_number_, voter_set_->id()).value(); - auto verifying_result = - ed_provider_->verify(vote.signature, payload, vote.id); - return verifying_result.has_value() and verifying_result.value(); + auto verify_ = [&](VoterSetId voter_set_id) { + auto payload = + scale::encode(vote.message, round_number_, voter_set_id).value(); + auto verifying_result = + ed_provider_->verify(vote.signature, payload, vote.id); + return verifying_result.has_value() and verifying_result.value(); + }; + auto res = verify_(voter_set_->id()); + if(res) return true; + auto logger = log::createLogger("VoteCryptoProvider", "authority"); + for (int i = -100; i < 100; i++) { + auto new_res = verify_(voter_set_->id() + i); + if (new_res) logger->info("Could've been correct with set id {}, actual {}", voter_set_->id() + i, voter_set_->id()); + } + return res; } bool VoteCryptoProviderImpl::verifyPrimaryPropose( diff --git a/core/consensus/grandpa/impl/voting_round_impl.cpp b/core/consensus/grandpa/impl/voting_round_impl.cpp index 68a88cd50c..bc9ca498b4 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.cpp +++ b/core/consensus/grandpa/impl/voting_round_impl.cpp @@ -862,7 +862,7 @@ namespace kagome::consensus::grandpa { return round_number_; } - MembershipCounter VotingRoundImpl::voterSetId() const { + VoterSetId VotingRoundImpl::voterSetId() const { return voter_set_->id(); } diff --git a/core/consensus/grandpa/impl/voting_round_impl.hpp b/core/consensus/grandpa/impl/voting_round_impl.hpp index eb7689ac2c..af77c0d429 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.hpp +++ b/core/consensus/grandpa/impl/voting_round_impl.hpp @@ -191,7 +191,7 @@ namespace kagome::consensus::grandpa { // Getters RoundNumber roundNumber() const override; - MembershipCounter voterSetId() const override; + VoterSetId voterSetId() const override; /** * Round is completable when we have block (stored in diff --git a/core/consensus/grandpa/structs.hpp b/core/consensus/grandpa/structs.hpp index ebf130c0ef..5c737e7a5b 100644 --- a/core/consensus/grandpa/structs.hpp +++ b/core/consensus/grandpa/structs.hpp @@ -174,7 +174,7 @@ namespace kagome::consensus::grandpa { // either prevote, precommit or primary propose struct VoteMessage { RoundNumber round_number{0}; - MembershipCounter counter{0}; + VoterSetId counter{0}; SignedMessage vote; Id id() const { diff --git a/core/consensus/grandpa/voter_set.cpp b/core/consensus/grandpa/voter_set.cpp index 633fe37e5e..84330fb8e5 100644 --- a/core/consensus/grandpa/voter_set.cpp +++ b/core/consensus/grandpa/voter_set.cpp @@ -22,7 +22,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::grandpa, VoterSet::Error, e) { namespace kagome::consensus::grandpa { - VoterSet::VoterSet(MembershipCounter id_of_set) : id_{id_of_set} {} + VoterSet::VoterSet(VoterSetId id_of_set) : id_{id_of_set} {} outcome::result VoterSet::insert(Id voter, size_t weight) { // zero authorities break the mapping logic a bit, but since they must not diff --git a/core/consensus/grandpa/voter_set.hpp b/core/consensus/grandpa/voter_set.hpp index cf7abdb340..2b95a6604b 100644 --- a/core/consensus/grandpa/voter_set.hpp +++ b/core/consensus/grandpa/voter_set.hpp @@ -30,7 +30,7 @@ namespace kagome::consensus::grandpa { VoterSet() = default; // for scale codec (in decode) - explicit VoterSet(MembershipCounter id_of_set); + explicit VoterSet(VoterSetId id_of_set); /** * Insert voter \param voter with \param weight @@ -40,7 +40,7 @@ namespace kagome::consensus::grandpa { /** * \return unique voter set membership id */ - inline MembershipCounter id() const { + inline VoterSetId id() const { return id_; } @@ -80,7 +80,7 @@ namespace kagome::consensus::grandpa { } private: - MembershipCounter id_{}; + VoterSetId id_{}; std::unordered_map map_; std::vector> list_; size_t total_weight_{0}; diff --git a/core/consensus/grandpa/voting_round.hpp b/core/consensus/grandpa/voting_round.hpp index 7d25db5fa3..b51cfd7a75 100644 --- a/core/consensus/grandpa/voting_round.hpp +++ b/core/consensus/grandpa/voting_round.hpp @@ -23,7 +23,7 @@ namespace kagome::consensus::grandpa { virtual RoundNumber roundNumber() const = 0; - virtual MembershipCounter voterSetId() const = 0; + virtual VoterSetId voterSetId() const = 0; virtual bool completable() const = 0; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 256b7c6fc5..1f7321dc7a 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -1439,9 +1439,12 @@ namespace { injector.template create>(); auto trie_storage = injector.template create>(); + auto authority_manager = + injector.template create>(); initialized.emplace(new application::mode::RecoveryMode( [&app_config, + authority_manager, storage = std::move(storage), header_repo = std::move(header_repo), trie_storage = std::move(trie_storage)] { @@ -1451,6 +1454,7 @@ namespace { storage, header_repo, trie_storage); + auto log = log::createLogger("RecoveryMode", "main"); if (res.has_error()) { SL_ERROR( @@ -1458,6 +1462,15 @@ namespace { log->flush(); return EXIT_FAILURE; } + auto number = + header_repo->getNumberById(app_config.recoverState().value()); + res = authority_manager->recalculateStoredState(number.value()); + if (res.has_error()) { + SL_ERROR( + log, "Recovery mode has failed: {}", res.error().message()); + log->flush(); + return EXIT_FAILURE; + } return EXIT_SUCCESS; })); diff --git a/core/network/peer_manager.hpp b/core/network/peer_manager.hpp index ddd7787ff3..257bc75ad0 100644 --- a/core/network/peer_manager.hpp +++ b/core/network/peer_manager.hpp @@ -22,7 +22,7 @@ namespace kagome::network { Roles roles = 0; BlockInfo best_block = {0, {}}; std::optional round_number = std::nullopt; - std::optional set_id = std::nullopt; + std::optional set_id = std::nullopt; BlockNumber last_finalized = 0; }; diff --git a/core/network/types/grandpa_message.hpp b/core/network/types/grandpa_message.hpp index 4a3cf787fe..ef2b65c8ea 100644 --- a/core/network/types/grandpa_message.hpp +++ b/core/network/types/grandpa_message.hpp @@ -7,19 +7,22 @@ #define KAGOME_NETWORK_GRANDPAMESSAGE #include + #include "consensus/grandpa/structs.hpp" +#include "consensus/grandpa/common.hpp" namespace kagome::network { using consensus::grandpa::BlockInfo; using consensus::grandpa::CompactCommit; using consensus::grandpa::GrandpaJustification; - using consensus::grandpa::MembershipCounter; + using consensus::grandpa::GrandpaJustification; using consensus::grandpa::RoundNumber; using consensus::grandpa::SignedMessage; using consensus::grandpa::SignedPrecommit; using consensus::grandpa::SignedPrevote; using consensus::grandpa::VoteMessage; + using consensus::grandpa::VoterSetId; using primitives::BlockNumber; struct GrandpaVote : public VoteMessage { @@ -35,7 +38,7 @@ namespace kagome::network { // The round this message is from. RoundNumber round{0}; // The voter set ID this message is from. - MembershipCounter set_id; + VoterSetId set_id; // The compact commit message. CompactCommit message; }; @@ -55,7 +58,7 @@ namespace kagome::network { struct GrandpaNeighborMessage { uint8_t version = 1; RoundNumber round_number; - MembershipCounter voter_set_id; + VoterSetId voter_set_id; BlockNumber last_finalized; }; @@ -77,7 +80,7 @@ namespace kagome::network { struct CatchUpRequest { RoundNumber round_number; - MembershipCounter voter_set_id; + VoterSetId voter_set_id; using Fingerprint = size_t; @@ -102,7 +105,7 @@ namespace kagome::network { } struct CatchUpResponse { - MembershipCounter voter_set_id{}; + VoterSetId voter_set_id{}; RoundNumber round_number{}; std::vector prevote_justification; std::vector precommit_justification; diff --git a/core/primitives/authority.hpp b/core/primitives/authority.hpp index 8b77b47042..4e1a53e15a 100644 --- a/core/primitives/authority.hpp +++ b/core/primitives/authority.hpp @@ -15,7 +15,7 @@ namespace kagome::primitives { using AuthorityWeight = uint64_t; - using AuthorityListId = uint64_t; + using AuthoritySetId = uint64_t; using AuthorityListSize = uint64_t; struct AuthorityId { @@ -105,15 +105,52 @@ namespace kagome::primitives { return s >> a.id >> a.weight; } - /// Special type for vector of authorities - struct AuthorityList : public std::vector { - AuthorityListId id{}; + /** + * List of authorities + */ + using AuthorityList = std::vector; + + /* + * List of authorities with an identifier + */ + struct AuthoritySet { + AuthoritySet() = default; + + AuthoritySet(AuthoritySetId id, AuthorityList authorities) + : id{id}, authorities{authorities} {} + + AuthoritySetId id{}; + AuthorityList authorities; - // Attention: When adding a member, we need to ensure correct - // destruction to avoid memory leaks or any other problem - using std::vector::vector; + auto begin() { + return authorities.begin(); + } + + auto end() { + return authorities.end(); + } + + auto begin() const { + return authorities.cbegin(); + } + + auto end() const { + return authorities.cend(); + } }; + template > + Stream &operator>>(Stream &s, AuthoritySet &a) { + return s >> a.id >> a.authorities; + } + + template > + Stream &operator<<(Stream &s, const AuthoritySet &a) { + return s << a.id << a.authorities; + } + } // namespace kagome::primitives #endif // KAGOME_AUTHORITY_HPP diff --git a/core/primitives/block_data.hpp b/core/primitives/block_data.hpp index 65769a5d7e..285008010b 100644 --- a/core/primitives/block_data.hpp +++ b/core/primitives/block_data.hpp @@ -32,7 +32,7 @@ namespace kagome::primitives { } static BlockDataFlags allUnset(primitives::BlockHash hash) { - return BlockDataFlags{std::move(hash), true, true, true, true, true}; + return BlockDataFlags{std::move(hash), false, false, false, false, false}; } primitives::BlockHash hash; diff --git a/core/primitives/digest.hpp b/core/primitives/digest.hpp index c399811c9d..cafdda8b6f 100644 --- a/core/primitives/digest.hpp +++ b/core/primitives/digest.hpp @@ -37,6 +37,15 @@ namespace kagome::primitives { ConsensusEngineId consensus_engine_id; common::Buffer data; + DigestItemCommon() { + consensus_engine_id = kGrandpaEngineId; + data = {}; + } + + DigestItemCommon(ConsensusEngineId consensus_engine_id, + common::Buffer data) + : consensus_engine_id(consensus_engine_id), data(std::move(data)) {} + bool operator==(const DigestItemCommon &rhs) const { return consensus_engine_id == rhs.consensus_engine_id and data == rhs.data; diff --git a/core/primitives/scheduled_change.hpp b/core/primitives/scheduled_change.hpp index 09a2a48b42..147b7722ce 100644 --- a/core/primitives/scheduled_change.hpp +++ b/core/primitives/scheduled_change.hpp @@ -42,6 +42,8 @@ namespace kagome::primitives { }; struct ForcedChange final : public AuthorityListChange { using AuthorityListChange::AuthorityListChange; + + BlockNumber delay_start; }; struct OnDisabled { uint32_t authority_index = 0; @@ -92,6 +94,11 @@ namespace kagome::primitives { Stream &operator>>(Stream &s, AuthorityListChange &alc) { return s >> alc.authorities >> alc.subchain_length; } + + template + Stream &operator>>(Stream &s, ForcedChange &change) { + return s >> change.delay_start >> change.authorities >> change.subchain_length; + } } // namespace kagome::primitives #endif // KAGOME_CORE_PRIMITIVES_SCHEDULED_CHANGE diff --git a/core/runtime/runtime_api/grandpa_api.hpp b/core/runtime/runtime_api/grandpa_api.hpp index 34e5a980bf..ba7fa2313f 100644 --- a/core/runtime/runtime_api/grandpa_api.hpp +++ b/core/runtime/runtime_api/grandpa_api.hpp @@ -12,6 +12,7 @@ #include "outcome/outcome.hpp" #include "primitives/authority.hpp" #include "primitives/block_id.hpp" +#include "primitives/common.hpp" #include "primitives/digest.hpp" #include "primitives/scheduled_change.hpp" #include "primitives/session_key.hpp" @@ -60,6 +61,12 @@ namespace kagome::runtime { */ virtual outcome::result authorities( const primitives::BlockId &block_id) = 0; + + /** + * @return the id of the current voter set at the provided block + */ + virtual outcome::result current_set_id( + const primitives::BlockHash &block) = 0; }; } // namespace kagome::runtime diff --git a/core/runtime/runtime_api/impl/grandpa_api.cpp b/core/runtime/runtime_api/impl/grandpa_api.cpp index bacce52839..788f6feb23 100644 --- a/core/runtime/runtime_api/impl/grandpa_api.cpp +++ b/core/runtime/runtime_api/impl/grandpa_api.cpp @@ -39,4 +39,10 @@ namespace kagome::runtime { "GrandpaApi_grandpa_authorities"); } + outcome::result GrandpaApiImpl::current_set_id( + const primitives::BlockHash &block_hash) { + return executor_->callAt( + block_hash, "GrandpaApi_current_set_id"); + } + } // namespace kagome::runtime diff --git a/core/runtime/runtime_api/impl/grandpa_api.hpp b/core/runtime/runtime_api/impl/grandpa_api.hpp index 303d09778d..e443ed4fbb 100644 --- a/core/runtime/runtime_api/impl/grandpa_api.hpp +++ b/core/runtime/runtime_api/impl/grandpa_api.hpp @@ -29,6 +29,9 @@ namespace kagome::runtime { outcome::result authorities( const primitives::BlockId &block_id) override; + outcome::result current_set_id( + const primitives::BlockHash &block) override; + private: std::shared_ptr block_header_repo_; std::shared_ptr executor_; @@ -36,4 +39,4 @@ namespace kagome::runtime { } // namespace kagome::runtime -#endif // KAGOME_RUNTIME_WAVM_GRANDPAAPI +#endif // KAGOME_RUNTIME_GRANDPAAPI diff --git a/core/storage/predefined_keys.hpp b/core/storage/predefined_keys.hpp index fcce11d29c..fa74f60c54 100644 --- a/core/storage/predefined_keys.hpp +++ b/core/storage/predefined_keys.hpp @@ -23,9 +23,6 @@ namespace kagome::storage { inline const common::Buffer kRuntimeHashesLookupKey = ":kagome:runtime_hashes"_buf; - inline const common::Buffer kSchedulerTreeLookupKey = - ":kagome:authorities:scheduler_tree"_buf; - inline const common::Buffer kOffchainWorkerStoragePrefix = ":kagome:ocw"_buf; inline const common::Buffer kChildStorageDefaultPrefix = diff --git a/core/storage/rocksdb/rocksdb_util.hpp b/core/storage/rocksdb/rocksdb_util.hpp index 0a61bbeb26..70cc75b55a 100644 --- a/core/storage/rocksdb/rocksdb_util.hpp +++ b/core/storage/rocksdb/rocksdb_util.hpp @@ -17,6 +17,8 @@ namespace kagome::storage { } if (s.IsIOError()) { + static log::Logger log = log::createLogger("LevelDb", "storage"); + SL_ERROR(log, ":{}", s.ToString()); return DatabaseError::IO_ERROR; } diff --git a/housekeeping/docker/kagome-dev/build_and_push.sh b/housekeeping/docker/kagome-dev/build_and_push.sh index b6d7303a94..02abdfd33d 100755 --- a/housekeeping/docker/kagome-dev/build_and_push.sh +++ b/housekeeping/docker/kagome-dev/build_and_push.sh @@ -2,7 +2,7 @@ cd "$(dirname "$0")" -VERSION=3 +VERSION=${VERSION:-3} TAG=soramitsu/kagome-dev:$VERSION docker build -t ${TAG}-minideb -f minideb.Dockerfile . diff --git a/housekeeping/docker/kagome-dev/minideb.Dockerfile b/housekeeping/docker/kagome-dev/minideb.Dockerfile index d46f37ba72..f0797fdd7e 100644 --- a/housekeeping/docker/kagome-dev/minideb.Dockerfile +++ b/housekeeping/docker/kagome-dev/minideb.Dockerfile @@ -27,7 +27,7 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o / "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-14 main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \ echo \ "deb http://deb.debian.org/debian/ testing main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ apt-get update && apt-get install --no-install-recommends -y \ docker-ce \ docker-ce-cli \ diff --git a/test/core/consensus/authority/CMakeLists.txt b/test/core/consensus/authority/CMakeLists.txt index 68656c9b61..62eb85a484 100644 --- a/test/core/consensus/authority/CMakeLists.txt +++ b/test/core/consensus/authority/CMakeLists.txt @@ -13,4 +13,5 @@ target_link_libraries(authority_manager_test buffer dummy_error logger_for_tests + in_memory_storage ) diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index 529d2d8c4f..a936738cd8 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -17,6 +17,7 @@ #include "mock/core/storage/trie/trie_storage_mock.hpp" #include "primitives/digest.hpp" #include "scale/scale.hpp" +#include "storage/in_memory/in_memory_storage.hpp" #include "storage/predefined_keys.hpp" #include "testutil/literals.hpp" #include "testutil/outcome.hpp" @@ -28,6 +29,7 @@ using authority::AuthorityManagerImpl; using authority::IsBlockFinalized; using kagome::storage::trie::EphemeralTrieBatchMock; using primitives::AuthorityList; +using primitives::AuthoritySet; using testing::_; using testing::Return; @@ -48,15 +50,19 @@ class AuthorityManagerTest : public testing::Test { void SetUp() override { app_state_manager = std::make_shared(); - authorities = std::make_shared(0); - authorities->emplace_back(makeAuthority("GenesisAuthority1", 5)); - authorities->emplace_back(makeAuthority("GenesisAuthority2", 10)); - authorities->emplace_back(makeAuthority("GenesisAuthority3", 15)); + AuthorityList authority_list; + authority_list.emplace_back(makeAuthority("GenesisAuthority1", 5)); + authority_list.emplace_back(makeAuthority("GenesisAuthority2", 10)); + authority_list.emplace_back(makeAuthority("GenesisAuthority3", 15)); + + authorities = std::make_shared(0, authority_list); block_tree = std::make_shared(); - storage = std::make_shared(); - EXPECT_CALL(*storage, getEphemeralBatchAt(_)) + persistent_storage = std::make_shared(); + + trie_storage = std::make_shared(); + EXPECT_CALL(*trie_storage, getEphemeralBatchAt(_)) .WillRepeatedly(testing::Invoke([] { auto batch = std::make_unique(); EXPECT_CALL(*batch, tryGet(_)) @@ -67,20 +73,27 @@ class AuthorityManagerTest : public testing::Test { grandpa_api = std::make_shared(); EXPECT_CALL(*grandpa_api, authorities(_)) - .WillRepeatedly(Return(*authorities)); + .WillRepeatedly(Return(authorities->authorities)); hasher = std::make_shared(); EXPECT_CALL(*hasher, twox_128(_)).WillRepeatedly(Return(common::Hash128{})); EXPECT_CALL(*app_state_manager, atPrepare(_)); + auto executore = std::make_shared(); + authority_manager = std::make_shared(AuthorityManagerImpl::Config{}, app_state_manager, block_tree, - storage, + trie_storage, grandpa_api, - hasher); + hasher, + persistent_storage); + + auto genesis_hash = "genesis"_hash256; + ON_CALL(*block_tree, getGenesisBlockHash()) + .WillByDefault(testing::ReturnRef(genesis_hash)); ON_CALL(*block_tree, hasDirectChain(_, _)) .WillByDefault(testing::Invoke([](auto &anc, auto &des) { @@ -149,11 +162,12 @@ class AuthorityManagerTest : public testing::Test { std::shared_ptr app_state_manager; std::shared_ptr block_tree; - std::shared_ptr storage; + std::shared_ptr trie_storage; + std::shared_ptr persistent_storage; std::shared_ptr grandpa_api; std::shared_ptr hasher; std::shared_ptr authority_manager; - std::shared_ptr authorities; + std::shared_ptr authorities; primitives::Authority makeAuthority(std::string_view id, uint32_t weight) { primitives::Authority authority; @@ -167,7 +181,7 @@ class AuthorityManagerTest : public testing::Test { /// Init by data from genesis config void prepareAuthorityManager() { auto node = authority::ScheduleNode::createAsRoot(genesis_block); - node->actual_authorities = authorities; + node->current_authorities = authorities; EXPECT_CALL(*block_tree, getLastFinalized()) .WillRepeatedly(Return(genesis_block)); @@ -189,7 +203,7 @@ class AuthorityManagerTest : public testing::Test { examining_block, IsBlockFinalized{false}); ASSERT_TRUE(actual_authorities_sptr.has_value()); const auto &actual_authorities = *actual_authorities_sptr.value(); - EXPECT_EQ(actual_authorities, expected_authorities); + EXPECT_EQ(actual_authorities.authorities, expected_authorities); } }; @@ -201,7 +215,7 @@ class AuthorityManagerTest : public testing::Test { TEST_F(AuthorityManagerTest, Init) { prepareAuthorityManager(); - examine({20, "D"_hash256}, *authorities); + examine({20, "D"_hash256}, authorities->authorities); } /** @@ -220,12 +234,12 @@ TEST_F(AuthorityManagerTest, Prune) { // Make expected state auto node = authority::ScheduleNode::createAsRoot({20, "D"_hash256}); - node->actual_authorities = - std::make_shared(orig_authorities); + node->current_authorities = + std::make_shared(orig_authorities); authority_manager->prune({20, "D"_hash256}); - examine({30, "F"_hash256}, orig_authorities); + examine({30, "F"_hash256}, orig_authorities.authorities); } /** @@ -253,11 +267,11 @@ TEST_F(AuthorityManagerTest, OnConsensus_ScheduledChange) { target_block, primitives::ScheduledChange(new_authorities, subchain_length))); - examine({5, "A"_hash256}, old_authorities); - examine({10, "B"_hash256}, old_authorities); - examine({15, "C"_hash256}, old_authorities); - examine({20, "D"_hash256}, old_authorities); - examine({25, "E"_hash256}, old_authorities); + examine({5, "A"_hash256}, old_authorities.authorities); + examine({10, "B"_hash256}, old_authorities.authorities); + examine({15, "C"_hash256}, old_authorities.authorities); + examine({20, "D"_hash256}, old_authorities.authorities); + examine({25, "E"_hash256}, old_authorities.authorities); authority_manager->prune({20, "D"_hash256}); @@ -289,9 +303,9 @@ TEST_F(AuthorityManagerTest, OnConsensus_ForcedChange) { target_block, primitives::ForcedChange(new_authorities, subchain_length))); - examine({5, "A"_hash256}, old_authorities); - examine({10, "B"_hash256}, old_authorities); - examine({15, "C"_hash256}, old_authorities); + examine({5, "A"_hash256}, old_authorities.authorities); + examine({10, "B"_hash256}, old_authorities.authorities); + examine({15, "C"_hash256}, old_authorities.authorities); examine({20, "D"_hash256}, new_authorities); examine({25, "E"_hash256}, new_authorities); } @@ -315,18 +329,18 @@ TEST_F(AuthorityManagerTest, DISABLED_OnConsensus_DisableAuthority) { primitives::BlockInfo target_block{10, "B"_hash256}; uint32_t authority_index = 1; - primitives::AuthorityList new_authorities = old_authorities; - assert(new_authorities.size() == 3); - new_authorities[authority_index].weight = 0; + primitives::AuthoritySet new_authorities = old_authorities; + assert(new_authorities.authorities.size() == 3); + new_authorities.authorities[authority_index].weight = 0; EXPECT_OUTCOME_SUCCESS( r1, authority_manager->onConsensus( target_block, primitives::OnDisabled({authority_index}))); - examine({5, "A"_hash256}, old_authorities); - examine({10, "B"_hash256}, new_authorities); - examine({15, "C"_hash256}, new_authorities); + examine({5, "A"_hash256}, old_authorities.authorities); + examine({10, "B"_hash256}, new_authorities.authorities); + examine({15, "C"_hash256}, new_authorities.authorities); } /** @@ -351,21 +365,21 @@ TEST_F(AuthorityManagerTest, OnConsensus_OnPause) { r1, authority_manager->onConsensus(target_block, primitives::Pause(delay))); - primitives::AuthorityList new_authorities = old_authorities; + primitives::AuthoritySet new_authorities = old_authorities; for (auto &authority : new_authorities) { authority.weight = 0; } - examine({5, "A"_hash256}, old_authorities); - examine({10, "B"_hash256}, old_authorities); - examine({15, "C"_hash256}, old_authorities); - examine({20, "D"_hash256}, old_authorities); - examine({25, "E"_hash256}, old_authorities); + examine({5, "A"_hash256}, old_authorities.authorities); + examine({10, "B"_hash256}, old_authorities.authorities); + examine({15, "C"_hash256}, old_authorities.authorities); + examine({20, "D"_hash256}, old_authorities.authorities); + examine({25, "E"_hash256}, old_authorities.authorities); authority_manager->prune({20, "D"_hash256}); - examine({20, "D"_hash256}, new_authorities); - examine({25, "E"_hash256}, new_authorities); + examine({20, "D"_hash256}, new_authorities.authorities); + examine({25, "E"_hash256}, new_authorities.authorities); } /** @@ -383,12 +397,12 @@ TEST_F(AuthorityManagerTest, OnConsensus_OnResume) { ASSERT_TRUE(old_auth_opt.has_value()); auto &enabled_authorities = *old_auth_opt.value(); - primitives::AuthorityList disabled_authorities = enabled_authorities; + primitives::AuthoritySet disabled_authorities = enabled_authorities; for (auto &authority : disabled_authorities) { authority.weight = 0; } - ASSERT_NE(enabled_authorities, disabled_authorities); + ASSERT_NE(enabled_authorities.authorities, disabled_authorities.authorities); { primitives::BlockInfo target_block{5, "A"_hash256}; @@ -401,10 +415,10 @@ TEST_F(AuthorityManagerTest, OnConsensus_OnResume) { authority_manager->prune({10, "B"_hash256}); } - examine({10, "B"_hash256}, disabled_authorities); - examine({15, "C"_hash256}, disabled_authorities); - examine({20, "D"_hash256}, disabled_authorities); - examine({25, "E"_hash256}, disabled_authorities); + examine({10, "B"_hash256}, disabled_authorities.authorities); + examine({15, "C"_hash256}, disabled_authorities.authorities); + examine({20, "D"_hash256}, disabled_authorities.authorities); + examine({25, "E"_hash256}, disabled_authorities.authorities); { primitives::BlockInfo target_block{15, "C"_hash256}; @@ -415,8 +429,9 @@ TEST_F(AuthorityManagerTest, OnConsensus_OnResume) { target_block, primitives::Resume(delay))); } - examine({10, "B"_hash256}, disabled_authorities); - examine({15, "C"_hash256}, disabled_authorities); - examine({20, "D"_hash256}, disabled_authorities); - examine({25, "E"_hash256}, enabled_authorities); + examine({10, "B"_hash256}, disabled_authorities.authorities); + examine({15, "C"_hash256}, disabled_authorities.authorities); + examine({20, "D"_hash256}, disabled_authorities.authorities); + examine({25, "E"_hash256}, enabled_authorities.authorities); } + diff --git a/test/mock/core/consensus/authority/authority_manager_mock.hpp b/test/mock/core/consensus/authority/authority_manager_mock.hpp index 3b934ed4ea..51b99df1dc 100644 --- a/test/mock/core/consensus/authority/authority_manager_mock.hpp +++ b/test/mock/core/consensus/authority/authority_manager_mock.hpp @@ -32,7 +32,8 @@ namespace kagome::authority { applyForcedChange, (const primitives::BlockInfo &, const primitives::AuthorityList &, - primitives::BlockNumber), + primitives::BlockNumber, + size_t), (override)); MOCK_METHOD(outcome::result, diff --git a/test/mock/core/consensus/grandpa/voting_round_mock.hpp b/test/mock/core/consensus/grandpa/voting_round_mock.hpp index 9a51631e09..a8039b754d 100644 --- a/test/mock/core/consensus/grandpa/voting_round_mock.hpp +++ b/test/mock/core/consensus/grandpa/voting_round_mock.hpp @@ -16,7 +16,7 @@ namespace kagome::consensus::grandpa { public: MOCK_METHOD(RoundNumber, roundNumber, (), (const, override)); - MOCK_METHOD(MembershipCounter, voterSetId, (), (const, override)); + MOCK_METHOD(VoterSetId, voterSetId, (), (const, override)); MOCK_METHOD(bool, completable, (), (const, override)); From 3107ffcbe113beca85b019caae5651ce19b075c4 Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 15 Aug 2022 15:56:46 +0300 Subject: [PATCH 35/74] Fix mocks --- .../impl/vote_crypto_provider_impl.cpp | 26 +++++++++++++------ .../impl/protocols/grandpa_protocol.hpp | 2 +- core/network/types/grandpa_message.hpp | 2 +- .../authority/authority_manager_test.cpp | 11 ++++---- .../authority/authority_manager_mock.hpp | 6 ++++- .../core/consensus/grandpa/chain_mock.hpp | 2 +- .../consensus/grandpa/environment_mock.hpp | 14 +++++----- 7 files changed, 38 insertions(+), 25 deletions(-) diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index 12441021e2..42ec0514f0 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -33,19 +33,29 @@ namespace kagome::consensus::grandpa { bool VoteCryptoProviderImpl::verify(const SignedMessage &vote, RoundNumber number) const { - auto verify_ = [&](VoterSetId voter_set_id) { - auto payload = - scale::encode(vote.message, round_number_, voter_set_id).value(); + auto verify_ = [&](VoterSetId voter_set_id, RoundNumber number) { + auto payload = scale::encode(vote.message, number, voter_set_id).value(); auto verifying_result = ed_provider_->verify(vote.signature, payload, vote.id); return verifying_result.has_value() and verifying_result.value(); }; - auto res = verify_(voter_set_->id()); - if(res) return true; + auto res = verify_(voter_set_->id(), round_number_); + if (res) return true; auto logger = log::createLogger("VoteCryptoProvider", "authority"); - for (int i = -100; i < 100; i++) { - auto new_res = verify_(voter_set_->id() + i); - if (new_res) logger->info("Could've been correct with set id {}, actual {}", voter_set_->id() + i, voter_set_->id()); + for (int i = -5; i < 5; i++) { + for (int j = -5; j < 5; j++) { + auto res = verify_(voter_set_->id() + i, round_number_ + j); + if (res) { + logger->debug( + "Could've been correct with round {} voter set {}, actual is " + "round {} voter set {}", + round_number_ + j, + voter_set_->id() + i, + round_number_, + voter_set_->id()); + return false; + } + } } return res; } diff --git a/core/network/impl/protocols/grandpa_protocol.hpp b/core/network/impl/protocols/grandpa_protocol.hpp index b702f5df10..50eb3c6a79 100644 --- a/core/network/impl/protocols/grandpa_protocol.hpp +++ b/core/network/impl/protocols/grandpa_protocol.hpp @@ -107,7 +107,7 @@ namespace kagome::network { std::shared_ptr scheduler_; std::set> + consensus::grandpa::VoterSetId>> recent_catchup_requests_by_round_; std::set recent_catchup_requests_by_peer_; diff --git a/core/network/types/grandpa_message.hpp b/core/network/types/grandpa_message.hpp index ef2b65c8ea..eb94e6ebbd 100644 --- a/core/network/types/grandpa_message.hpp +++ b/core/network/types/grandpa_message.hpp @@ -87,7 +87,7 @@ namespace kagome::network { inline Fingerprint fingerprint() const { auto result = std::hash()(round_number); - boost::hash_combine(result, std::hash()(voter_set_id)); + boost::hash_combine(result, std::hash()(voter_set_id)); return result; }; }; diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index a936738cd8..e8f4620512 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -180,8 +180,8 @@ class AuthorityManagerTest : public testing::Test { /// Init by data from genesis config void prepareAuthorityManager() { - auto node = authority::ScheduleNode::createAsRoot(genesis_block); - node->current_authorities = authorities; + auto node = + authority::ScheduleNode::createAsRoot(authorities, genesis_block); EXPECT_CALL(*block_tree, getLastFinalized()) .WillRepeatedly(Return(genesis_block)); @@ -233,9 +233,9 @@ TEST_F(AuthorityManagerTest, Prune) { auto &orig_authorities = *authorities_opt.value(); // Make expected state - auto node = authority::ScheduleNode::createAsRoot({20, "D"_hash256}); - node->current_authorities = - std::make_shared(orig_authorities); + auto node = authority::ScheduleNode::createAsRoot( + std::make_shared(orig_authorities), + {20, "D"_hash256}); authority_manager->prune({20, "D"_hash256}); @@ -434,4 +434,3 @@ TEST_F(AuthorityManagerTest, OnConsensus_OnResume) { examine({20, "D"_hash256}, disabled_authorities.authorities); examine({25, "E"_hash256}, enabled_authorities.authorities); } - diff --git a/test/mock/core/consensus/authority/authority_manager_mock.hpp b/test/mock/core/consensus/authority/authority_manager_mock.hpp index 51b99df1dc..8a049a8d2c 100644 --- a/test/mock/core/consensus/authority/authority_manager_mock.hpp +++ b/test/mock/core/consensus/authority/authority_manager_mock.hpp @@ -16,7 +16,7 @@ namespace kagome::authority { struct AuthorityManagerMock : public AuthorityManager { MOCK_METHOD(primitives::BlockInfo, base, (), (const, override)); - MOCK_METHOD(std::optional>, + MOCK_METHOD(std::optional>, authorities, (const primitives::BlockInfo &, IsBlockFinalized), (const, override)); @@ -52,6 +52,10 @@ namespace kagome::authority { (override)); MOCK_METHOD(void, prune, (const primitives::BlockInfo &block), (override)); + + MOCK_METHOD(outcome::result, + recalculateStoredState, + (primitives::BlockNumber last_finalized_number)); }; } // namespace kagome::authority diff --git a/test/mock/core/consensus/grandpa/chain_mock.hpp b/test/mock/core/consensus/grandpa/chain_mock.hpp index 46e61bc37f..b9b6dfd364 100644 --- a/test/mock/core/consensus/grandpa/chain_mock.hpp +++ b/test/mock/core/consensus/grandpa/chain_mock.hpp @@ -31,7 +31,7 @@ namespace kagome::consensus::grandpa { MOCK_METHOD(outcome::result, bestChainContaining, - (const BlockHash &, std::optional set_id), + (const BlockHash &, std::optional set_id), (const, override)); }; diff --git a/test/mock/core/consensus/grandpa/environment_mock.hpp b/test/mock/core/consensus/grandpa/environment_mock.hpp index 322cb71355..5de88fdfb4 100644 --- a/test/mock/core/consensus/grandpa/environment_mock.hpp +++ b/test/mock/core/consensus/grandpa/environment_mock.hpp @@ -23,7 +23,7 @@ namespace kagome::consensus::grandpa { MOCK_METHOD(outcome::result, onVoted, (RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, const SignedMessage &propose), (override)); @@ -31,13 +31,13 @@ namespace kagome::consensus::grandpa { sendState, (const libp2p::peer::PeerId &peer_id, const MovableRoundState &state, - MembershipCounter set_id), + VoterSetId set_id), (override)); MOCK_METHOD(outcome::result, onCommitted, (RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, const BlockInfo &vote, const GrandpaJustification &justification), (override)); @@ -45,7 +45,7 @@ namespace kagome::consensus::grandpa { MOCK_METHOD(outcome::result, onNeighborMessageSent, (RoundNumber round, - MembershipCounter set_id, + VoterSetId set_id, BlockNumber last_finalized), (override)); @@ -57,7 +57,7 @@ namespace kagome::consensus::grandpa { MOCK_METHOD(outcome::result, finalize, - (MembershipCounter id, + (VoterSetId id, const GrandpaJustification &justification), (override)); @@ -69,14 +69,14 @@ namespace kagome::consensus::grandpa { MOCK_METHOD(outcome::result, onCatchUpRequested, (const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number), (override)); MOCK_METHOD(outcome::result, onCatchUpRespond, (const libp2p::peer::PeerId &peer_id, - MembershipCounter set_id, + VoterSetId set_id, RoundNumber round_number, std::vector prevote_justification, std::vector precommit_justification, From 2a53eea567845944db3bbd5e53e343912bfac84a Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 13:33:31 +0300 Subject: [PATCH 36/74] fix: block data flags Signed-off-by: Dmitriy Khaustov aka xDimon --- core/primitives/block_data.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/primitives/block_data.hpp b/core/primitives/block_data.hpp index 65769a5d7e..285008010b 100644 --- a/core/primitives/block_data.hpp +++ b/core/primitives/block_data.hpp @@ -32,7 +32,7 @@ namespace kagome::primitives { } static BlockDataFlags allUnset(primitives::BlockHash hash) { - return BlockDataFlags{std::move(hash), true, true, true, true, true}; + return BlockDataFlags{std::move(hash), false, false, false, false, false}; } primitives::BlockHash hash; From 989ab078674e63430b9e583a6032c09157936807 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 14:14:15 +0300 Subject: [PATCH 37/74] fix: assert Signed-off-by: Dmitriy Khaustov aka xDimon --- core/runtime/runtime_api/impl/core.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/runtime/runtime_api/impl/core.cpp b/core/runtime/runtime_api/impl/core.cpp index 2114b3d5b3..8c191138e2 100644 --- a/core/runtime/runtime_api/impl/core.cpp +++ b/core/runtime/runtime_api/impl/core.cpp @@ -34,7 +34,7 @@ namespace kagome::runtime { outcome::result CoreImpl::execute_block( const primitives::Block &block) { - BOOST_ASSERT([] { + BOOST_ASSERT([&] { auto parent_res = header_repo_->getBlockHeader(block.header.parent_hash); return parent_res.has_value() and parent_res.value().number == block.header.number - 1; From ce570b27f622659b362958ed3f9df50dde228e5a Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 15:50:10 +0300 Subject: [PATCH 38/74] temp: way to save state of authmngr Signed-off-by: Dmitriy Khaustov aka xDimon --- .../authority/impl/authority_manager_impl.cpp | 53 ++++++++++++--- .../authority/impl/authority_manager_impl.hpp | 8 ++- .../authority/impl/schedule_node.hpp | 66 +++++++++++++++++++ 3 files changed, 118 insertions(+), 9 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index bedbaedee1..69747d8e66 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -35,19 +35,22 @@ namespace kagome::authority { std::shared_ptr block_tree, std::shared_ptr trie_storage, std::shared_ptr grandpa_api, - std::shared_ptr hasher) + std::shared_ptr hasher, + std::shared_ptr buffer_storage) : config_{std::move(config)}, header_repo_(std::move(header_repo)), block_tree_(std::move(block_tree)), trie_storage_(std::move(trie_storage)), grandpa_api_(std::move(grandpa_api)), hasher_(std::move(hasher)), + buffer_storage_(std::move(buffer_storage)), log_{log::createLogger("AuthorityManager", "authority")} { BOOST_ASSERT(header_repo_ != nullptr); BOOST_ASSERT(block_tree_ != nullptr); BOOST_ASSERT(grandpa_api_ != nullptr); BOOST_ASSERT(trie_storage_ != nullptr); BOOST_ASSERT(hasher_ != nullptr); + BOOST_ASSERT(buffer_storage_ != nullptr); BOOST_ASSERT(app_state_manager != nullptr); app_state_manager->atPrepare([&] { return prepare(); }); @@ -225,6 +228,10 @@ namespace kagome::authority { auto &val = UNIQUE_NAME(expr_r_).value(); bool AuthorityManagerImpl::prepare() { + if (load()) { + return true; + } + const auto finalized_block = block_tree_->getLastFinalized(); PREPARE_TRY( @@ -677,7 +684,7 @@ namespace kagome::authority { } if (message.consensus_engine_id == primitives::kGrandpaEngineId) { OUTCOME_TRY(decoded, message.decode()); - return visit_in_place( + auto res = visit_in_place( decoded.asGrandpaDigest(), [this, &block]( const primitives::ScheduledChange &msg) -> outcome::result { @@ -700,6 +707,18 @@ namespace kagome::authority { [](auto &) { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; }); + save(); + return res; + } else if (message.consensus_engine_id + == primitives::kUnsupportedEngineId_POL1 + or message.consensus_engine_id + == primitives::kUnsupportedEngineId_BEEF) { + SL_DEBUG(log_, + "Unsupported consensus engine id in block {}: {}", + block, + message.consensus_engine_id.toString()); + return outcome::success(); + } else { SL_WARN(log_, "Unknown consensus engine id in block {}: {}", @@ -709,6 +728,20 @@ namespace kagome::authority { } } + void AuthorityManagerImpl::save() { + auto data = scale::encode(root_).value(); + buffer_storage_->put(Buffer::fromString("authmngrdata"), Buffer(data)).value(); + } + + bool AuthorityManagerImpl::load() { + auto r = buffer_storage_->tryLoad(Buffer::fromString("authmngrdata")).value(); + if (r.has_value()) { + root_ = scale::decode>(r.value()).value(); + return true; + } + return false; + } + void AuthorityManagerImpl::prune(const primitives::BlockInfo &block) { if (block == root_->block) { return; @@ -742,6 +775,8 @@ namespace kagome::authority { } SL_VERBOSE(log_, "Prune authority manager upto block {}", block); + + save(); } std::shared_ptr AuthorityManagerImpl::getAppropriateAncestor( @@ -911,12 +946,14 @@ namespace kagome::authority { prune(block_info); } - if (found) { - SL_VERBOSE(log_, - "{} headers are observed and {} digests are applied (+{})", - block_number, - count, - delta); + if (not(block_number % 250000)) + // if (found) + { + SL_INFO(log_, + "{} headers are observed and {} digests are applied (+{})", + block_number, + count, + delta); delta = 0; } } diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 29f521d9ea..8783280c66 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -11,6 +11,7 @@ #include "crypto/hasher.hpp" #include "log/logger.hpp" +#include "storage/buffer_map_types.hpp" namespace kagome::application { class AppStateManager; @@ -55,7 +56,8 @@ namespace kagome::authority { std::shared_ptr block_tree, std::shared_ptr trie_storage, std::shared_ptr grandpa_api, - std::shared_ptr hash); + std::shared_ptr hash, + std::shared_ptr buffer_storage); ~AuthorityManagerImpl() override = default; @@ -98,6 +100,9 @@ namespace kagome::authority { void prune(const primitives::BlockInfo &block) override; private: + bool load(); + void save(); + /** * @brief Find schedule_node according to the block * @param block for which to find the schedule node @@ -130,6 +135,7 @@ namespace kagome::authority { std::shared_ptr trie_storage_; std::shared_ptr grandpa_api_; std::shared_ptr hasher_; + std::shared_ptr buffer_storage_; std::shared_ptr root_; log::Logger log_; diff --git a/core/consensus/authority/impl/schedule_node.hpp b/core/consensus/authority/impl/schedule_node.hpp index 79fd29233a..d7dbe0de15 100644 --- a/core/consensus/authority/impl/schedule_node.hpp +++ b/core/consensus/authority/impl/schedule_node.hpp @@ -71,6 +71,72 @@ namespace kagome::authority { primitives::BlockNumber resume_for = INACTIVE; }; + template > + Stream &operator<<(Stream &s, const ScheduleNode &b) { + s << b.block << b.actual_authorities << b.actual_authorities->id + << b.enabled; + if (b.scheduled_after != ScheduleNode::INACTIVE) { + s << b.scheduled_after << b.scheduled_authorities + << b.scheduled_authorities->id; + } else { + s << static_cast(0); + } + if (b.forced_for != ScheduleNode::INACTIVE) { + s << b.forced_for << b.forced_authorities << b.forced_authorities->id; + } else { + s << static_cast(0); + } + if (b.pause_after != ScheduleNode::INACTIVE) { + s << b.pause_after; + } else { + s << static_cast(0); + } + if (b.resume_for != ScheduleNode::INACTIVE) { + s << b.resume_for; + } else { + s << static_cast(0); + } + s << b.descendants; + return s; + } + + template > + Stream &operator>>(Stream &s, ScheduleNode &b) { + s >> const_cast(b.block); // NOLINT + s >> b.actual_authorities; + s >> const_cast(b.actual_authorities->id); // NOLINT + s >> b.enabled; + primitives::BlockNumber bn; + if (s >> bn, bn) { + b.scheduled_after = bn; + s >> b.scheduled_authorities; + s >> const_cast(b.scheduled_authorities->id); // NOLINT + } else { + b.scheduled_after = ScheduleNode::INACTIVE; + } + if (s >> bn, bn) { + b.forced_for = bn; + s >> b.forced_authorities; + s >> const_cast(b.forced_authorities->id); // NOLINT + } else { + b.forced_for = ScheduleNode::INACTIVE; + } + if (s >> bn, bn) { + b.pause_after = bn; + } else { + b.pause_after = ScheduleNode::INACTIVE; + } + if (s >> bn, bn) { + b.resume_for = bn; + } else { + b.resume_for = ScheduleNode::INACTIVE; + } + s >> b.descendants; + return s; + } + } // namespace kagome::authority #endif // KAGOME_CONSENSUS_AUTHORITIES_SCHEDULE_NODE From b82f40b2a08680a08a0c5544d91aac3ae06d778b Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 15:50:45 +0300 Subject: [PATCH 39/74] fix: remove useless assert Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/impl/babe_impl.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 239f237fc2..569b8ab493 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -386,7 +386,6 @@ namespace kagome::consensus::babe { void BabeImpl::startCatchUp(const libp2p::peer::PeerId &peer_id, const primitives::BlockInfo &target_block) { - BOOST_ASSERT(current_state_ != Babe::State::HEADERS_LOADING); BOOST_ASSERT(current_state_ != Babe::State::STATE_LOADING); // synchronize missing blocks with their bodies From 18f82dd0038058e935d3ebf922d406827e9b3a84 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 16:25:39 +0300 Subject: [PATCH 40/74] fix: minor issues Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/impl/block_appender_impl.cpp | 2 +- core/consensus/babe/impl/block_appender_impl.hpp | 2 +- core/network/impl/synchronizer_impl.cpp | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 6fec590597..cfbf9cb3f2 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -283,7 +283,7 @@ namespace kagome::consensus { auto block_delta = block_info.number - speed_data_.block_number; auto time_delta = now - speed_data_.time; - if (block_delta >= 20000 or time_delta >= std::chrono::minutes(1)) { + if (block_delta >= 10000 or time_delta >= std::chrono::minutes(1)) { SL_INFO(logger_, "Imported {} more headers of blocks. Average speed is {} bps", block_delta, diff --git a/core/consensus/babe/impl/block_appender_impl.hpp b/core/consensus/babe/impl/block_appender_impl.hpp index b03fee7c9f..498708411f 100644 --- a/core/consensus/babe/impl/block_appender_impl.hpp +++ b/core/consensus/babe/impl/block_appender_impl.hpp @@ -68,7 +68,7 @@ namespace kagome::consensus { struct { std::chrono::high_resolution_clock::time_point time; primitives::BlockNumber block_number; - } speed_data_; + } speed_data_ = {}; log::Logger logger_; }; diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index c461602631..34704ce7e5 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -232,7 +232,7 @@ namespace kagome::network { BOOST_ASSERT(lower < upper); // Callback what will be called at the end of finding the best common block - Synchronizer::SyncResultHandler find_handler = + auto find_handler = [wp = weak_from_this(), peer_id, handler = std::move(handler)]( outcome::result res) mutable { if (auto self = wp.lock()) { @@ -891,6 +891,11 @@ namespace kagome::network { const primitives::BlockInfo &block, const std::vector &keys, SyncResultHandler &&handler) { + SL_TRACE(log_, + "SyncState requested to {} for block {} with {} keys", + peer_id, + block, + keys.size()); if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { // execute if not already started or iteration continues if (not state_syncing_.load() @@ -977,7 +982,7 @@ namespace kagome::network { } } - // not well formed way to place 0th batch key to front + // not well-formed way to place 0th batch key to front std::map keymap; for (const auto &[_, val] : self->batches_store_) { unsigned i = std::get<1>(val); From 990b7c19513bce9b76fc13b42687b2b3a738212b Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Mon, 15 Aug 2022 23:15:43 +0300 Subject: [PATCH 41/74] fix: sync state Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/babe.hpp | 2 + core/consensus/babe/impl/babe_impl.cpp | 30 ++- .../babe/impl/block_appender_impl.cpp | 2 +- core/network/impl/synchronizer_impl.cpp | 210 ++++++++++-------- core/network/impl/synchronizer_impl.hpp | 7 +- core/network/synchronizer.hpp | 2 - 6 files changed, 144 insertions(+), 109 deletions(-) diff --git a/core/consensus/babe/babe.hpp b/core/consensus/babe/babe.hpp index e4105da5e8..3c3db1ca2f 100644 --- a/core/consensus/babe/babe.hpp +++ b/core/consensus/babe/babe.hpp @@ -26,6 +26,8 @@ namespace kagome::consensus::babe { WAIT_REMOTE_STATUS, // Node is just launched and waits status of remote // peer to sync missing blocks HEADERS_LOADING, // Fast sync requested; phase of headers downloading + HEADERS_LOADED, // Fast sync requested; headers downloaded, ready to + // syncing of state STATE_LOADING, // Fast sync requested; phase of state downloading CATCHING_UP, // Node recognized the missing blocks and started fetching // blocks between the best missing one and one of the diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 569b8ab493..b3f3d61ade 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -151,7 +151,7 @@ namespace kagome::consensus::babe { case SyncMethod::Fast: // Has uncompleted downloading state - if (false) { // FIXME + if (false) { // FIXME // Start loading of state current_state_ = State::STATE_LOADING; } else { @@ -299,7 +299,12 @@ namespace kagome::consensus::babe { const auto ¤t_best_block = current_best_block_res.value(); if (current_best_block == status.best_block) { - onSynchronized(); + if (current_state_ == Babe::State::HEADERS_LOADING) { + current_state_ = Babe::State::HEADERS_LOADED; + startStateSyncing(peer_id); + } else if (current_state_ == Babe::State::CATCHING_UP) { + onSynchronized(); + } return; } @@ -332,7 +337,8 @@ namespace kagome::consensus::babe { } // Start catching up if gap recognized - if (current_state_ == Babe::State::SYNCHRONIZED) { + if (current_state_ == Babe::State::SYNCHRONIZED + or current_state_ == Babe::State::HEADERS_LOADED) { if (announce.header.number > current_best_block.number + 1) { auto block_hash = hasher_->blake2b_256(scale::encode(announce.header).value()); @@ -351,8 +357,6 @@ namespace kagome::consensus::babe { [wp = weak_from_this(), announce = announce, peer_id]( outcome::result block_res) mutable { if (auto self = wp.lock()) { - self->synchronizer_->endSync(); - if (block_res.has_error()) { return; } @@ -360,6 +364,7 @@ namespace kagome::consensus::babe { // Headers are loaded; Start to sync state if (self->current_state_ == Babe::State::HEADERS_LOADING) { + self->current_state_ = Babe::State::HEADERS_LOADED; self->startStateSyncing(peer_id); return; } @@ -395,7 +400,6 @@ namespace kagome::consensus::babe { [wp = weak_from_this(), block = target_block, peer_id]( outcome::result res) { if (auto self = wp.lock()) { - self->synchronizer_->endSync(); if (res.has_error()) { SL_DEBUG(self->log_, "Catching up {} to block {} is failed: {}", @@ -417,15 +421,21 @@ namespace kagome::consensus::babe { if (is_ran) { SL_VERBOSE( log_, "Catching up {} to block {} is ran", peer_id, target_block); - current_state_ = State::CATCHING_UP; + if (current_state_ == State::HEADERS_LOADED) { + current_state_ = State::HEADERS_LOADING; + } else if (current_state_ == State::WAIT_BLOCK_ANNOUNCE + or current_state_ == State::WAIT_REMOTE_STATUS + or current_state_ == State::SYNCHRONIZED) { + current_state_ = State::CATCHING_UP; + } } } void BabeImpl::startStateSyncing(const libp2p::peer::PeerId &peer_id) { - BOOST_ASSERT(current_state_ == Babe::State::HEADERS_LOADING + BOOST_ASSERT(current_state_ == Babe::State::HEADERS_LOADED or current_state_ == Babe::State::STATE_LOADING); - if (current_state_ == Babe::State::HEADERS_LOADING - or current_state_ == Babe::State::STATE_LOADING) { + if (current_state_ != Babe::State::HEADERS_LOADED + and current_state_ != Babe::State::STATE_LOADING) { SL_WARN(log_, "Syncing of state can not be start: Bad state of babe"); return; } diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index cfbf9cb3f2..3e092268a1 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -161,7 +161,7 @@ namespace kagome::consensus { SL_DEBUG( logger_, - "Applying block {} ({} in slot {}, epoch {})", // + "Appending header of block {} ({} in slot {}, epoch {})", block_info, babe_header.slotType() == SlotType::Primary ? "primary" : babe_header.slotType() == SlotType::SecondaryVRF ? "secondary-vrf" diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 34704ce7e5..2bc335450a 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -167,11 +167,6 @@ namespace kagome::network { const libp2p::peer::PeerId &peer_id, Synchronizer::SyncResultHandler &&handler, bool subscribe_to_block) { - if (not state_syncing_.load() && not syncing_.load()) { - syncing_.store(true); - } else { - return false; - } // Subscribe on demand if (subscribe_to_block) { subscribeToBlock(block_info, std::move(handler)); @@ -280,11 +275,6 @@ namespace kagome::network { const primitives::BlockHeader &header, const libp2p::peer::PeerId &peer_id, Synchronizer::SyncResultHandler &&handler) { - if (not state_syncing_.load() && not syncing_.load()) { - syncing_.store(true); - } else { - return false; - } auto block_hash = hasher_->blake2b_256(scale::encode(header).value()); const primitives::BlockInfo block_info(header.number, block_hash); @@ -891,37 +881,50 @@ namespace kagome::network { const primitives::BlockInfo &block, const std::vector &keys, SyncResultHandler &&handler) { + bool bool_val = false; + if (not state_sync_request_in_progress_.compare_exchange_strong(bool_val, + true)) { + SL_TRACE(log_, + "SyncState was not requested to {} for block {} with {} keys: " + "previous request in progress", + peer_id, + block, + keys.size()); + return; + } + + state_syncing_ = true; + SL_TRACE(log_, "SyncState requested to {} for block {} with {} keys", peer_id, block, keys.size()); - if (sync_method_ == application::AppConfiguration::SyncMethod::Fast) { - // execute if not already started or iteration continues - if (not state_syncing_.load() - || (not keys.empty() && not keys[0].empty())) { - state_syncing_.store(true); - if (keys[0].empty()) { - sync_block_ = block; - } - network::StateRequest request{block.hash, keys, true}; - auto protocol = router_->getStateProtocol(); - BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); + // TODO watch to sync only one block + // if (keys[0].empty()) { + // sync_block_ = block; + // } + + network::StateRequest request{block.hash, keys, true}; + + auto protocol = router_->getStateProtocol(); + BOOST_ASSERT_MSG(protocol, "Router did not provide state protocol"); - SL_TRACE(log_, "State syncing started."); + SL_TRACE(log_, "State syncing started."); - auto response_handler = [wp = weak_from_this(), - block, - peer_id, - handler = std::move(handler)]( - auto &&response_res) mutable { + auto response_handler = + [wp = weak_from_this(), block, peer_id, handler = std::move(handler)]( + auto &&response_res) mutable { auto self = wp.lock(); if (not self) { return; } + // Request failed if (response_res.has_error()) { + self->state_sync_request_in_progress_ = false; + SL_WARN(self->log_, "State syncing failed with error: {}", response_res.error().message()); @@ -929,6 +932,7 @@ namespace kagome::network { return; } + // Processing of response for (unsigned i = 0; i < response_res.value().entries.size(); ++i) { const auto &response = response_res.value().entries[i]; @@ -961,17 +965,43 @@ namespace kagome::network { } } + // Handle completion of syncing if (response.complete) { auto res = batch->commit(); - SL_INFO( - self->log_, - "{} syncing finished. Root hash: {}. {}.", - i ? "Child state" : "State", - res.value().toHex(), - res.value() == response.state_root ? "Match" : "Don't match"); - if (res.value() != response.state_root) { - SL_INFO( - self->log_, "Should be {}", response.state_root.toHex()); + if (res.has_value()) { + if (i != 0) { // Child state + if (res.value() == response.state_root) { + SL_INFO(self->log_, + "Syncing of child state has finished. " + "Root hashes match: {}", + response.state_root); + } else { + SL_WARN(self->log_, + "Syncing of child state has finished. " + "Root hashes mismatch: expected={}, actual={}", + response.state_root, + res.value()); + } + } else { // Main state + auto header_res = + self->block_tree_->getBlockHeader(block.hash); + BOOST_ASSERT_MSG( + header_res.has_value(), + "It is state of existing block; head must be existing"); + const auto &header = header_res.value(); + if (res.value() == response.state_root) { + SL_INFO(self->log_, + "Syncing of child state has finished. " + "Root hashes match: {}", + response.state_root); + } else { + SL_WARN(self->log_, + "Syncing of child state has finished. " + "Root hashes mismatch: expected={}, actual={}", + header.state_root, + res.value()); + } + } } self->trie_changes_tracker_->onBlockAdded(block.hash); } @@ -995,29 +1025,25 @@ namespace kagome::network { keys.push_back(val); } - if (response_res.value().entries[0].complete) { - self->sync_method_ = - application::AppConfiguration::SyncMethod::Full; - if (handler) { - handler(block); - self->state_syncing_.store(false); - } - } else { + self->state_sync_request_in_progress_ = false; + + if (not response_res.value().entries[0].complete) { SL_TRACE(self->log_, "State syncing continues. {} entries loaded", self->entries_); self->syncState(peer_id, block, keys, std::move(handler)); + return; + } + + // State syncing has completed; Switch to the full syncing + self->sync_method_ = application::AppConfiguration::SyncMethod::Full; + if (handler) { + handler(block); + self->state_syncing_ = false; } }; - protocol->request( - peer_id, std::move(request), std::move(response_handler)); - } - } else { - if (handler) { - handler(block); - } - } + protocol->request(peer_id, std::move(request), std::move(response_handler)); } void SynchronizerImpl::applyNextBlock() { @@ -1067,6 +1093,10 @@ namespace kagome::network { } } + const BlockInfo block_info(block.header->number, block.hash); + + // Skip applied and finalized blocks and + // discard side-chain below last finalized if (block.header->number <= last_finalized_block.number) { auto header_res = block_tree_->getBlockHeader(hash); if (not header_res.has_value()) { @@ -1074,55 +1104,53 @@ namespace kagome::network { SL_WARN( log_, "Block {} {} not applied as discarded", - BlockInfo(number, hash), + block_info, n ? fmt::format("and {} others have", n) : fmt::format("has")); if (handler) handler(Error::DISCARDED_BLOCK); } + } else { - const BlockInfo block_info(block.header->number, block.hash); + outcome::result applying_res = outcome::success(); + + if (sync_method_ == application::AppConfiguration::SyncMethod::Full) { + applying_res = block_executor_->applyBlock(std::move(block)); + + } else if (not state_syncing_) { + applying_res = block_appender_->appendBlock(std::move(block)); - if (sync_method_ == application::AppConfiguration::SyncMethod::Full - && sync_block_ && block_info.number <= sync_block_.value().number) { - SL_WARN( - log_, "Skip {} till fast synchronized block", block_info.number); - applyNextBlock(); } else { - auto applying_res = - sync_method_ == application::AppConfiguration::SyncMethod::Full - ? block_executor_->applyBlock(std::move(block)) - : block_appender_->appendBlock(std::move(block)); - - if (sync_method_ == application::AppConfiguration::SyncMethod::Full - && sync_block_ - && block_info.number == sync_block_.value().number + 1) { - sync_block_ = std::nullopt; - } + auto n = discardBlock(block.hash); + SL_WARN( + log_, + "Block {} {} not applied as discarded: state syncing on block in " + "progress", + block_info, + n ? fmt::format("and {} others have", n) : fmt::format("has")); + if (handler) handler(Error::DISCARDED_BLOCK); + } - notifySubscribers({number, hash}, applying_res); - - if (not applying_res.has_value()) { - if (applying_res - != outcome::failure(blockchain::BlockTreeError::BLOCK_EXISTS)) { - notifySubscribers({number, hash}, applying_res.as_failure()); - auto n = discardBlock(block_info.hash); - SL_WARN( - log_, - "Block {} {} been discarded: {}", - block_info, - n ? fmt::format("and {} others have", n) : fmt::format("has"), - applying_res.error().message()); - if (handler) handler(Error::DISCARDED_BLOCK); - } else { - SL_DEBUG(log_, "Block {} is skipped as existing", block_info); - if (handler) handler(block_info); - // applyNextBlock(); - } + notifySubscribers(block_info, applying_res); + + if (not applying_res.has_value()) { + if (applying_res + != outcome::failure(blockchain::BlockTreeError::BLOCK_EXISTS)) { + notifySubscribers(block_info, applying_res.as_failure()); + auto n = discardBlock(block.hash); + SL_WARN( + log_, + "Block {} {} been discarded: {}", + block_info, + n ? fmt::format("and {} others have", n) : fmt::format("has"), + applying_res.error().message()); + if (handler) handler(Error::DISCARDED_BLOCK); } else { - telemetry_->notifyBlockImported( - block_info, telemetry::BlockOrigin::kNetworkInitialSync); + SL_DEBUG(log_, "Block {} is skipped as existing", block_info); if (handler) handler(block_info); - // applyNextBlock(); } + } else { + telemetry_->notifyBlockImported( + block_info, telemetry::BlockOrigin::kNetworkInitialSync); + if (handler) handler(block_info); } } } diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 64afead9bb..f662fe62ce 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -149,10 +149,6 @@ namespace kagome::network { std::optional limit, SyncResultHandler &&handler); - void endSync() override { - syncing_.store(false); - } - private: /// Subscribes handler for block with provided {@param block_info} /// {@param handler} will be called When block is received or discarded @@ -206,8 +202,9 @@ namespace kagome::network { log::Logger log_ = log::createLogger("Synchronizer", "synchronizer"); telemetry::Telemetry telemetry_ = telemetry::createTelemetryService(); - std::atomic_bool syncing_ = false; std::atomic_bool state_syncing_ = false; + std::atomic_bool state_sync_request_in_progress_ = false; + bool node_is_shutting_down_ = false; struct KnownBlock { diff --git a/core/network/synchronizer.hpp b/core/network/synchronizer.hpp index bd2301d674..34bfe4a96c 100644 --- a/core/network/synchronizer.hpp +++ b/core/network/synchronizer.hpp @@ -53,8 +53,6 @@ namespace kagome::network { const primitives::BlockInfo &block, const std::vector &keys, SyncResultHandler &&handler) = 0; - - virtual void endSync() = 0; }; } // namespace kagome::network From dda60117cd40bb286d8e986c255b272f47d26bda Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 16 Aug 2022 12:11:41 +0300 Subject: [PATCH 42/74] fix: sync state Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/synchronizer_impl.cpp | 63 ++++++++++++------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 2bc335450a..1ccb80967b 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -969,40 +969,37 @@ namespace kagome::network { if (response.complete) { auto res = batch->commit(); if (res.has_value()) { - if (i != 0) { // Child state - if (res.value() == response.state_root) { - SL_INFO(self->log_, - "Syncing of child state has finished. " - "Root hashes match: {}", - response.state_root); - } else { - SL_WARN(self->log_, - "Syncing of child state has finished. " - "Root hashes mismatch: expected={}, actual={}", - response.state_root, - res.value()); - } - } else { // Main state - auto header_res = - self->block_tree_->getBlockHeader(block.hash); - BOOST_ASSERT_MSG( - header_res.has_value(), - "It is state of existing block; head must be existing"); - const auto &header = header_res.value(); - if (res.value() == response.state_root) { - SL_INFO(self->log_, - "Syncing of child state has finished. " - "Root hashes match: {}", - response.state_root); - } else { - SL_WARN(self->log_, - "Syncing of child state has finished. " - "Root hashes mismatch: expected={}, actual={}", - header.state_root, - res.value()); + const auto &expected = [&] { + if (i != 0) { // Child state + return response.state_root; + } else { // Main state + auto header_res = + self->block_tree_->getBlockHeader(block.hash); + BOOST_ASSERT_MSG( + header_res.has_value(), + "It is state of existing block; head must be existing"); + const auto &header = header_res.value(); + return header.state_root; } + }(); + const auto &actual = res.value(); + + if (actual == expected) { + SL_INFO(self->log_, + "Syncing of {}state has finished. " + "Root hashes match: {}", + i != 0 ? "child " : "", + actual); + } else { + SL_WARN(self->log_, + "Syncing of {}state has finished. " + "Root hashes mismatch: expected={}, actual={}", + i != 0 ? "child " : "", + expected, + actual); } } + self->trie_changes_tracker_->onBlockAdded(block.hash); } @@ -1081,7 +1078,7 @@ namespace kagome::network { if (node) { auto &block = node.mapped().data; BOOST_ASSERT(block.header.has_value()); - auto number = block.header->number; + const BlockInfo block_info(block.header->number, block.hash); const auto &last_finalized_block = block_tree_->getLastFinalized(); @@ -1093,8 +1090,6 @@ namespace kagome::network { } } - const BlockInfo block_info(block.header->number, block.hash); - // Skip applied and finalized blocks and // discard side-chain below last finalized if (block.header->number <= last_finalized_block.number) { From 1a551818c43c351521ce77e765159a3e16e8d646 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 16 Aug 2022 12:26:22 +0300 Subject: [PATCH 43/74] refactor: level log messages of state protocol Signed-off-by: Dmitriy Khaustov aka xDimon --- core/network/impl/protocols/state_protocol_impl.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/network/impl/protocols/state_protocol_impl.cpp b/core/network/impl/protocols/state_protocol_impl.cpp index 0a12961698..4cf9abcc11 100644 --- a/core/network/impl/protocols/state_protocol_impl.cpp +++ b/core/network/impl/protocols/state_protocol_impl.cpp @@ -185,7 +185,7 @@ namespace kagome::network { return; } - SL_WARN(self->log_, + SL_DEBUG(self->log_, "Established outgoing {} stream with {}", self->protocol_, stream->remotePeerId().value()); @@ -203,7 +203,7 @@ namespace kagome::network { return; } - SL_WARN(self->log_, "State request sent"); + SL_DEBUG(self->log_, "State request sent"); if (not write_res.has_value()) { SL_WARN(self->log_, "Error getting response"); @@ -296,7 +296,7 @@ namespace kagome::network { std::function)> &&response_handler) { auto read_writer = std::make_shared(stream); - SL_WARN(log_, + SL_DEBUG(log_, "Read response from outgoing {} stream with {}", protocol_, stream->remotePeerId().value()); From d63e548a9451827ce470d6e7bb526d0f409bd2f2 Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 16 Aug 2022 20:27:39 +0300 Subject: [PATCH 44/74] fix: remove non finalized block before state syncing start Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/impl/babe_impl.cpp | 38 ++++++++++++++++++++++++- core/consensus/babe/impl/babe_impl.hpp | 5 +++- core/injector/application_injector.cpp | 3 +- core/network/impl/synchronizer_impl.cpp | 6 ++-- 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index b3f3d61ade..d2ba9fc583 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -13,6 +13,7 @@ #include "blockchain/block_tree_error.hpp" #include "common/buffer.hpp" #include "consensus/babe/babe_error.hpp" +#include "consensus/babe/consistency_keeper.hpp" #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/parachains_inherent_data.hpp" #include "consensus/babe/impl/threshold_util.hpp" @@ -55,7 +56,8 @@ namespace kagome::consensus::babe { authority_update_observer, std::shared_ptr synchronizer, std::shared_ptr babe_util, - std::shared_ptr offchain_worker_api) + std::shared_ptr offchain_worker_api, + std::shared_ptr consistency_keeper) : app_config_(app_config), lottery_{std::move(lottery)}, babe_configuration_{std::move(configuration)}, @@ -71,6 +73,7 @@ namespace kagome::consensus::babe { synchronizer_(std::move(synchronizer)), babe_util_(std::move(babe_util)), offchain_worker_api_(std::move(offchain_worker_api)), + consistency_keeper_(std::move(consistency_keeper)), log_{log::createLogger("Babe", "babe")}, telemetry_{telemetry::createTelemetryService()} { BOOST_ASSERT(lottery_); @@ -86,6 +89,7 @@ namespace kagome::consensus::babe { BOOST_ASSERT(synchronizer_); BOOST_ASSERT(babe_util_); BOOST_ASSERT(offchain_worker_api_); + BOOST_ASSERT(consistency_keeper_); BOOST_ASSERT(app_state_manager); @@ -445,6 +449,38 @@ namespace kagome::consensus::babe { // Switch to last finalized to have a state on it auto block_at_state = block_tree_->getLastFinalized(); + // Next do-while-loop serves for removal non finalized blocks + bool affected; + do { + affected = false; + + auto block_tree_leaves = block_tree_->getLeaves(); + + for (const auto &hash : block_tree_leaves) { + if (hash == block_at_state.hash) continue; + + auto header_res = block_tree_->getBlockHeader(hash); + if (header_res.has_error()) { + SL_CRITICAL(log_, + "Can't get header of one of removing leave_block: {}", + header_res.error().message()); + continue; + } + + const auto &header = header_res.value(); + + // Block below last finalized must not being. Don't touch just in case + if (header.number < block_at_state.number) { + continue; + } + + std::ignore = consistency_keeper_->start( + primitives::BlockInfo(header.number, hash)); + + affected = true; + } + } while (affected); + SL_TRACE(log_, "Trying to sync state on block {} from {}", block_at_state, diff --git a/core/consensus/babe/impl/babe_impl.hpp b/core/consensus/babe/impl/babe_impl.hpp index 4c2683cd21..a527ff38a8 100644 --- a/core/consensus/babe/impl/babe_impl.hpp +++ b/core/consensus/babe/impl/babe_impl.hpp @@ -45,6 +45,7 @@ namespace kagome::runtime { } namespace kagome::consensus::babe { + class ConsistencyKeeper; inline const auto kTimestampId = primitives::InherentIdentifier::fromString("timstap0").value(); @@ -81,7 +82,8 @@ namespace kagome::consensus::babe { authority_update_observer, std::shared_ptr synchronizer, std::shared_ptr babe_util, - std::shared_ptr offchain_worker_api); + std::shared_ptr offchain_worker_api, + std::shared_ptr consistency_keeper); ~BabeImpl() override = default; @@ -172,6 +174,7 @@ namespace kagome::consensus::babe { std::shared_ptr synchronizer_; std::shared_ptr babe_util_; std::shared_ptr offchain_worker_api_; + std::shared_ptr consistency_keeper_; State current_state_{State::WAIT_REMOTE_STATUS}; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index dec11019ce..8509d4784e 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -1412,7 +1412,8 @@ namespace { injector.template create>(), injector.template create>(), injector.template create>(), - injector.template create>()); + injector.template create>(), + injector.template create>()); auto protocol_factory = injector.template create>(); diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 1ccb80967b..440f6ce6b9 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -986,15 +986,17 @@ namespace kagome::network { if (actual == expected) { SL_INFO(self->log_, - "Syncing of {}state has finished. " + "Syncing of {}state on block {} has finished. " "Root hashes match: {}", i != 0 ? "child " : "", + block, actual); } else { SL_WARN(self->log_, - "Syncing of {}state has finished. " + "Syncing of {}state on block {} has finished. " "Root hashes mismatch: expected={}, actual={}", i != 0 ? "child " : "", + block, expected, actual); } From 598701032c9ff3e896d9851bd4700a4f5473506c Mon Sep 17 00:00:00 2001 From: Dmitriy Khaustov aka xDimon Date: Tue, 16 Aug 2022 20:36:35 +0300 Subject: [PATCH 45/74] fix: add log to view last finalized before rolling back his descendants Signed-off-by: Dmitriy Khaustov aka xDimon --- core/consensus/babe/impl/babe_impl.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index d2ba9fc583..1618c4ad74 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -449,6 +449,10 @@ namespace kagome::consensus::babe { // Switch to last finalized to have a state on it auto block_at_state = block_tree_->getLastFinalized(); + SL_DEBUG(log_, + "Rolling back non-finalized blocks. Last known finalized is {}", + block_at_state); + // Next do-while-loop serves for removal non finalized blocks bool affected; do { From 28dd79722a5f7cba146f0ea89432a3b010c38c62 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 18 Aug 2022 23:48:09 +0300 Subject: [PATCH 46/74] Fixes in authority manager --- .../authority/authority_manager_error.cpp | 4 +- .../authority/authority_manager_error.hpp | 1 + .../authority/impl/authority_manager_impl.cpp | 59 +++++++++++++------ .../impl/vote_crypto_provider_impl.cpp | 6 ++ core/injector/application_injector.cpp | 1 + 5 files changed, 52 insertions(+), 19 deletions(-) diff --git a/core/consensus/authority/authority_manager_error.cpp b/core/consensus/authority/authority_manager_error.cpp index bfdcf07feb..c2b2060ac0 100644 --- a/core/consensus/authority/authority_manager_error.cpp +++ b/core/consensus/authority/authority_manager_error.cpp @@ -11,9 +11,11 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::authority, AuthorityManagerError, e) { case E::UNKNOWN_ENGINE_ID: return "Unknown engine_id"; case E::ORPHAN_BLOCK_OR_ALREADY_FINALIZED: - return "Block it not descendant of last finalized block"; + return "Block is not descendant of last finalized block"; case E::CAN_NOT_SAVE_STATE: return "Can not save state"; + case E::CANT_RECALCULATE_ON_PRUNED_STATE: + return "Can't recalculate authority set ids on a pruned database"; } return "unknown error (invalid AuthorityManagerError)"; } diff --git a/core/consensus/authority/authority_manager_error.hpp b/core/consensus/authority/authority_manager_error.hpp index 0771ba6dd1..f07cca162b 100644 --- a/core/consensus/authority/authority_manager_error.hpp +++ b/core/consensus/authority/authority_manager_error.hpp @@ -13,6 +13,7 @@ namespace kagome::authority { UNKNOWN_ENGINE_ID = 1, ORPHAN_BLOCK_OR_ALREADY_FINALIZED, CAN_NOT_SAVE_STATE, + CANT_RECALCULATE_ON_PRUNED_STATE, }; } diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index af867eef67..d67625ff40 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -78,7 +78,8 @@ namespace kagome::authority { break; } - OUTCOME_TRY(header, block_tree.getBlockHeader(hash)); + // just obtained from block tree + auto header = block_tree.getBlockHeader(hash).value(); // observe possible changes of authorities for (auto &digest_item : boost::adaptors::reverse(header.digest)) { @@ -142,6 +143,12 @@ namespace kagome::authority { return outcome::success(); } + outcome::result clearScheduleGraphRoot( + storage::BufferStorage &storage) { + OUTCOME_TRY(storage.remove(kScheduleGraphRootKey)); + return outcome::success(); + } + static const common::Buffer kForcedSetIdListKey = common::Buffer::fromString(":authority_manager:forced_set_id_list"); @@ -164,7 +171,12 @@ namespace kagome::authority { bool is_unapplied_change = false; for (auto hash = finalized_block.hash; !found_set_change;) { - OUTCOME_TRY(header, block_tree.getBlockHeader(hash)); + auto header_res = block_tree.getBlockHeader(hash); + if (!header_res) { + SL_ERROR( + log, "Failed to obtain the last finalized block header {}", hash); + } + OUTCOME_TRY(header, header_res); if (header.number == 0) { found_set_change = true; @@ -373,22 +385,32 @@ namespace kagome::authority { std::make_shared(0, initial_authorities), {0, genesis_hash}); + // if state is pruned + if (header_repo_->getBlockHeader(1).has_error()) { + SL_WARN(log_, + "Can't recalculate authority set id on a prune state, fall back " + "to fetching from runtime"); + return clearScheduleGraphRoot(*persistent_storage_); + } + for (primitives::BlockNumber number = 0; number <= last_finalized_number; number++) { auto start = std::chrono::steady_clock::now(); + auto header_res = header_repo_->getBlockHeader(number); + if(!header_res) continue; // Temporary workaround about the justification pruning bug + auto& header = header_res.value(); + OUTCOME_TRY(hash, header_repo_->getHashByNumber(number)); - OUTCOME_TRY(header, header_repo_->getBlockHeader(number)); primitives::BlockInfo info{number, hash}; - bool has_authority_change = false; for (auto &msg : header.digest) { if (auto consensus_msg = boost::get(&msg); consensus_msg != nullptr) { - OUTCOME_TRY(onConsensus(info, *consensus_msg)); - has_authority_change = true; + onConsensus(info, *consensus_msg).value(); } } - if (has_authority_change) prune(info); + auto justification_res = block_tree_->getBlockJustification(hash); + if (justification_res.has_value()) prune(info); auto end = std::chrono::steady_clock::now(); SL_TRACE( log_, @@ -413,7 +435,7 @@ namespace kagome::authority { IsBlockFinalized finalized) const { auto node = getAppropriateAncestor(target_block); - if (not node) { + if (node == nullptr) { return std::nullopt; } @@ -422,6 +444,9 @@ namespace kagome::authority { ? (bool)finalized : node->current_block.number <= block_tree_->getLastFinalized().number; + if (target_block.number == 1514496) { + node_in_finalized_chain = true; + } auto adjusted_node = node->makeDescendant(target_block, node_in_finalized_chain); @@ -432,6 +457,7 @@ namespace kagome::authority { "Pick authority set with id {} for block {}", adjusted_node->current_authorities->id, target_block); + // Since getAppropriateAncestor worked normally on this block auto header = block_tree_->getBlockHeader(target_block.hash).value(); auto id_from_storage = fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header) @@ -499,11 +525,7 @@ namespace kagome::authority { return outcome::success(); }; - KAGOME_PROFILE_START(is_ancestor_node_finalized) IsBlockFinalized is_ancestor_node_finalized = true; - ancestor_node->current_block.number - <= block_tree_->getLastFinalized().number; - KAGOME_PROFILE_END(is_ancestor_node_finalized) if (ancestor_node->current_block == block) { ancestor_node->adjust(is_ancestor_node_finalized); @@ -544,8 +566,12 @@ namespace kagome::authority { delay, current_block, delay_start + delay); - OUTCOME_TRY(delay_start_header, - block_tree_->getBlockHeader(delay_start + 1)); + auto delay_start_header_res = + block_tree_->getBlockHeader(delay_start + 1); + if (delay_start_header_res.has_error()) { + SL_ERROR(log_, "Failed to obtain header by number {}", delay_start + 1); + } + OUTCOME_TRY(delay_start_header, delay_start_header_res); auto ancestor_node = getAppropriateAncestor({delay_start, delay_start_header.parent_hash}); @@ -592,11 +618,8 @@ namespace kagome::authority { return outcome::success(); }; - OUTCOME_TRY(delay_start_child, - block_tree_->getBlockHeader(delay_start + 1)); - auto new_node = ancestor_node->makeDescendant( - {delay_start, delay_start_child.parent_hash}, true); + {delay_start, delay_start_header.parent_hash}, true); OUTCOME_TRY(force_change(new_node)); diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index 42ec0514f0..ec53d5ff46 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -41,7 +41,13 @@ namespace kagome::consensus::grandpa { }; auto res = verify_(voter_set_->id(), round_number_); if (res) return true; + auto logger = log::createLogger("VoteCryptoProvider", "authority"); + logger->error( + "round {} voter set {}", + round_number_, + voter_set_->id()); + for (int i = -5; i < 5; i++) { for (int j = -5; j < 5; j++) { auto res = verify_(voter_set_->id() + i, round_number_ + j); diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 1f7321dc7a..52f8220b2e 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -1462,6 +1462,7 @@ namespace { log->flush(); return EXIT_FAILURE; } + auto number = header_repo->getNumberById(app_config.recoverState().value()); res = authority_manager->recalculateStoredState(number.value()); From 782f0ce51380295c58cc7152fb30d8897e2252d5 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 14 Jul 2022 11:28:31 +0300 Subject: [PATCH 47/74] Add todo for schedule change processing --- core/consensus/authority/impl/authority_manager_impl.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index d67625ff40..9a3beb1ae7 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -525,6 +525,8 @@ namespace kagome::authority { return outcome::success(); }; + /// TODO(Harrm): Should account for finality when fetching an authority set + /// for some purposes, but not when scheduling further changes IsBlockFinalized is_ancestor_node_finalized = true; if (ancestor_node->current_block == block) { From 89d15b585a0fe46a87230e2f57bc635735acf9cf Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 14 Jul 2022 11:31:43 +0300 Subject: [PATCH 48/74] Remove debug code from verify() --- .../consensus/authority/authority_manager.hpp | 2 -- .../impl/vote_crypto_provider_impl.cpp | 36 +++---------------- 2 files changed, 5 insertions(+), 33 deletions(-) diff --git a/core/consensus/authority/authority_manager.hpp b/core/consensus/authority/authority_manager.hpp index 00f80b61d6..43e51e9484 100644 --- a/core/consensus/authority/authority_manager.hpp +++ b/core/consensus/authority/authority_manager.hpp @@ -20,8 +20,6 @@ namespace kagome::authority { public: virtual ~AuthorityManager() = default; - using HeaderIterator = std::function; - /** * Recalculate the authority change graph starting from genesis and up to * the last finalized block. The result shall be stored in the provided diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index ec53d5ff46..88baf3c21f 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -33,37 +33,11 @@ namespace kagome::consensus::grandpa { bool VoteCryptoProviderImpl::verify(const SignedMessage &vote, RoundNumber number) const { - auto verify_ = [&](VoterSetId voter_set_id, RoundNumber number) { - auto payload = scale::encode(vote.message, number, voter_set_id).value(); - auto verifying_result = - ed_provider_->verify(vote.signature, payload, vote.id); - return verifying_result.has_value() and verifying_result.value(); - }; - auto res = verify_(voter_set_->id(), round_number_); - if (res) return true; - - auto logger = log::createLogger("VoteCryptoProvider", "authority"); - logger->error( - "round {} voter set {}", - round_number_, - voter_set_->id()); - - for (int i = -5; i < 5; i++) { - for (int j = -5; j < 5; j++) { - auto res = verify_(voter_set_->id() + i, round_number_ + j); - if (res) { - logger->debug( - "Could've been correct with round {} voter set {}, actual is " - "round {} voter set {}", - round_number_ + j, - voter_set_->id() + i, - round_number_, - voter_set_->id()); - return false; - } - } - } - return res; + auto payload = + scale::encode(vote.message, number, voter_set_->id()).value(); + auto verifying_result = + ed_provider_->verify(vote.signature, payload, vote.id); + return verifying_result.has_value() and verifying_result.value(); } bool VoteCryptoProviderImpl::verifyPrimaryPropose( From 83eaa6c98f03fd6ff179f3511d728a1c80089e04 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 14 Jul 2022 11:33:31 +0300 Subject: [PATCH 49/74] Remove commented code --- core/primitives/digest.hpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/primitives/digest.hpp b/core/primitives/digest.hpp index cafdda8b6f..8eca282c84 100644 --- a/core/primitives/digest.hpp +++ b/core/primitives/digest.hpp @@ -37,14 +37,7 @@ namespace kagome::primitives { ConsensusEngineId consensus_engine_id; common::Buffer data; - DigestItemCommon() { - consensus_engine_id = kGrandpaEngineId; - data = {}; - } - - DigestItemCommon(ConsensusEngineId consensus_engine_id, - common::Buffer data) - : consensus_engine_id(consensus_engine_id), data(std::move(data)) {} + DigestItemCommon() = default; bool operator==(const DigestItemCommon &rhs) const { return consensus_engine_id == rhs.consensus_engine_id From 2a91f55c9bb5c8acd57dd7ae6a285c90dd9f72bd Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 15 Jul 2022 12:04:11 +0300 Subject: [PATCH 50/74] Fixes from review --- core/consensus/authority/authority_manager.hpp | 4 ++-- core/consensus/grandpa/environment.hpp | 3 +-- core/network/types/grandpa_message.hpp | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/consensus/authority/authority_manager.hpp b/core/consensus/authority/authority_manager.hpp index 43e51e9484..787b1dd6be 100644 --- a/core/consensus/authority/authority_manager.hpp +++ b/core/consensus/authority/authority_manager.hpp @@ -24,10 +24,10 @@ namespace kagome::authority { * Recalculate the authority change graph starting from genesis and up to * the last finalized block. The result shall be stored in the provided * storage. This operation may take a considerable amount of time. - * @param header_iter - iterator over finalized block headers * @return nothing on success, error otherwise */ - virtual outcome::result recalculateStoredState(primitives::BlockNumber last_finalized_number) = 0; + virtual outcome::result recalculateStoredState( + primitives::BlockNumber last_finalized_number) = 0; /** * @return block associated with the root of scheduled changes tree diff --git a/core/consensus/grandpa/environment.hpp b/core/consensus/grandpa/environment.hpp index 27d7aa6215..39021a561c 100644 --- a/core/consensus/grandpa/environment.hpp +++ b/core/consensus/grandpa/environment.hpp @@ -104,8 +104,7 @@ namespace kagome::consensus::grandpa { * @param last_finalized last known finalized block */ virtual outcome::result onNeighborMessageSent( - RoundNumber round, VoterSetId set_id, - BlockNumber last_finalized) = 0; + RoundNumber round, VoterSetId set_id, BlockNumber last_finalized) = 0; /** * Validate provided {@param justification} for finalization {@param block}. diff --git a/core/network/types/grandpa_message.hpp b/core/network/types/grandpa_message.hpp index eb94e6ebbd..56b2b7182e 100644 --- a/core/network/types/grandpa_message.hpp +++ b/core/network/types/grandpa_message.hpp @@ -8,15 +8,14 @@ #include -#include "consensus/grandpa/structs.hpp" #include "consensus/grandpa/common.hpp" +#include "consensus/grandpa/structs.hpp" namespace kagome::network { using consensus::grandpa::BlockInfo; using consensus::grandpa::CompactCommit; using consensus::grandpa::GrandpaJustification; - using consensus::grandpa::GrandpaJustification; using consensus::grandpa::RoundNumber; using consensus::grandpa::SignedMessage; using consensus::grandpa::SignedPrecommit; From 5a0bd7c9c1d0df9b8b37e2f5d89b9d9017ad96e4 Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 15 Jul 2022 12:22:09 +0300 Subject: [PATCH 51/74] Fix recovery logging --- .../authority/impl/authority_manager_impl.cpp | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 9a3beb1ae7..8038bc6cb5 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -384,7 +384,8 @@ namespace kagome::authority { root_ = ScheduleNode::createAsRoot( std::make_shared(0, initial_authorities), {0, genesis_hash}); - + SL_INFO(log_, + "Recovering authority manager state... (might take a few minutes)"); // if state is pruned if (header_repo_->getBlockHeader(1).has_error()) { SL_WARN(log_, @@ -393,6 +394,7 @@ namespace kagome::authority { return clearScheduleGraphRoot(*persistent_storage_); } + auto start = std::chrono::steady_clock::now(); for (primitives::BlockNumber number = 0; number <= last_finalized_number; number++) { auto start = std::chrono::steady_clock::now(); @@ -412,12 +414,17 @@ namespace kagome::authority { auto justification_res = block_tree_->getBlockJustification(hash); if (justification_res.has_value()) prune(info); auto end = std::chrono::steady_clock::now(); - SL_TRACE( - log_, - "Process block #{} in {} ms", - number, - std::chrono::duration_cast(end - start) - .count()); + auto duration = end - start; + using namespace std::chrono_literals; + // 5 seconds is nothing special, just a random more-or-like convenient + // duration. + if (duration > 5s) { + SL_VERBOSE(log_, + "Processed {} out of {} blocks", + number, + last_finalized_number); + start = end; + } } return outcome::success(); } @@ -507,10 +514,11 @@ namespace kagome::authority { node->action = ScheduleNode::ScheduledChange{activate_at, new_authorities}; - SL_VERBOSE(log_, - "Change is scheduled after block #{} (set id={})", - activate_at, - new_authorities->id); + SL_VERBOSE( + log_, + "Authority set change is scheduled after block #{} (set id={})", + activate_at, + new_authorities->id); size_t index = 0; for (auto &authority : *new_authorities) { @@ -839,7 +847,7 @@ namespace kagome::authority { } storeScheduleGraphRoot(*persistent_storage_, *root_).value(); - SL_VERBOSE(log_, "Prune authority manager upto block {}", block); + SL_DEBUG(log_, "Prune authority manager upto block {}", block); } std::shared_ptr AuthorityManagerImpl::getAppropriateAncestor( From 9ca0c4ac77637d08241130d4a00d0e4e2683816b Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 15 Jul 2022 12:44:00 +0300 Subject: [PATCH 52/74] Make NoAction alias Empty --- core/consensus/authority/impl/schedule_node.hpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/core/consensus/authority/impl/schedule_node.hpp b/core/consensus/authority/impl/schedule_node.hpp index c5640bb866..4bd7d70106 100644 --- a/core/consensus/authority/impl/schedule_node.hpp +++ b/core/consensus/authority/impl/schedule_node.hpp @@ -8,6 +8,7 @@ #include +#include "common/empty.hpp" #include "common/tagged.hpp" #include "primitives/authority.hpp" @@ -42,17 +43,7 @@ namespace kagome::authority { std::shared_ptr makeDescendant( const primitives::BlockInfo &block, IsBlockFinalized finalized) const; - struct NoAction { - friend inline ::scale::ScaleEncoderStream &operator<<( - ::scale::ScaleEncoderStream &s, const NoAction &change) { - return s; - } - - friend inline ::scale::ScaleDecoderStream &operator>>( - ::scale::ScaleDecoderStream &s, NoAction &change) { - return s; - } - }; + using NoAction = Empty; struct ScheduledChange { primitives::BlockNumber applied_block{}; From e1cc1b4f9743fc21c4e2580efe2f92afe2b109e2 Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 15 Jul 2022 18:18:53 +0300 Subject: [PATCH 53/74] Fix authority manager tests --- .../authority/impl/authority_manager_impl.cpp | 4 ++-- core/primitives/scheduled_change.hpp | 18 ++++++++++++++++-- .../authority/authority_manager_test.cpp | 10 +++++++--- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 8038bc6cb5..8b21f48d83 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -583,7 +583,7 @@ namespace kagome::authority { } OUTCOME_TRY(delay_start_header, delay_start_header_res); auto ancestor_node = - getAppropriateAncestor({delay_start, delay_start_header.parent_hash}); + getAppropriateAncestor({delay_start, delay_start_hash}); if (not ancestor_node) { return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; @@ -629,7 +629,7 @@ namespace kagome::authority { }; auto new_node = ancestor_node->makeDescendant( - {delay_start, delay_start_header.parent_hash}, true); + {delay_start, delay_start_hash}, true); OUTCOME_TRY(force_change(new_node)); diff --git a/core/primitives/scheduled_change.hpp b/core/primitives/scheduled_change.hpp index 147b7722ce..f4f6e53e1a 100644 --- a/core/primitives/scheduled_change.hpp +++ b/core/primitives/scheduled_change.hpp @@ -40,8 +40,14 @@ namespace kagome::primitives { struct ScheduledChange final : public AuthorityListChange { using AuthorityListChange::AuthorityListChange; }; + struct ForcedChange final : public AuthorityListChange { - using AuthorityListChange::AuthorityListChange; + ForcedChange() = default; + + ForcedChange(AuthorityList authorities, + uint32_t delay, + BlockNumber delay_start) + : AuthorityListChange(authorities, delay), delay_start{delay_start} {} BlockNumber delay_start; }; @@ -97,8 +103,16 @@ namespace kagome::primitives { template Stream &operator>>(Stream &s, ForcedChange &change) { - return s >> change.delay_start >> change.authorities >> change.subchain_length; + return s >> change.delay_start >> change.authorities + >> change.subchain_length; } + + template + Stream &operator<<(Stream &s, const ForcedChange &change) { + return s << change.delay_start << change.authorities + << change.subchain_length; + } + } // namespace kagome::primitives #endif // KAGOME_CORE_PRIMITIVES_SCHEDULED_CHANGE diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index e8f4620512..945dbfbfa6 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -162,6 +162,7 @@ class AuthorityManagerTest : public testing::Test { std::shared_ptr app_state_manager; std::shared_ptr block_tree; + std::shared_ptr header_repo; std::shared_ptr trie_storage; std::shared_ptr persistent_storage; std::shared_ptr grandpa_api; @@ -294,18 +295,21 @@ TEST_F(AuthorityManagerTest, OnConsensus_ForcedChange) { auto &old_authorities = *old_auth_opt.value(); primitives::BlockInfo target_block{10, "B"_hash256}; + EXPECT_CALL(*header_repo, getHashByNumber(target_block.number)) + .WillOnce(Return(target_block.hash)); primitives::AuthorityList new_authorities{makeAuthority("Auth1", 123)}; - uint32_t subchain_length = 10; + uint32_t subchain_length = 5; EXPECT_OUTCOME_SUCCESS( r1, authority_manager->onConsensus( target_block, - primitives::ForcedChange(new_authorities, subchain_length))); + primitives::ForcedChange( + new_authorities, subchain_length, target_block.number))); examine({5, "A"_hash256}, old_authorities.authorities); examine({10, "B"_hash256}, old_authorities.authorities); - examine({15, "C"_hash256}, old_authorities.authorities); + examine({15, "C"_hash256}, new_authorities); examine({20, "D"_hash256}, new_authorities); examine({25, "E"_hash256}, new_authorities); } From ef97ba2f8e1798298499d3b00162dbfe60eec85d Mon Sep 17 00:00:00 2001 From: Harrm Date: Sat, 16 Jul 2022 11:31:45 +0300 Subject: [PATCH 54/74] Fix alleged stack-use-after-free --- core/consensus/authority/impl/authority_manager_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 8b21f48d83..19c0adf2f0 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -285,7 +285,7 @@ namespace kagome::authority { root_->current_authorities->id); } else if (last_finalized_block.number == 0) { - auto genesis_hash = block_tree_->getGenesisBlockHash(); + auto& genesis_hash = block_tree_->getGenesisBlockHash(); PREPARE_TRY(initial_authorities, grandpa_api_->authorities(genesis_hash), "Can't get grandpa authorities for genesis block: {}", From 0627676eda488384d569ecacfad8575e63d53de5 Mon Sep 17 00:00:00 2001 From: Harrm Date: Sun, 17 Jul 2022 20:00:59 +0300 Subject: [PATCH 55/74] Fix force change recovery problem --- core/consensus/authority/impl/authority_manager_impl.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 19c0adf2f0..33d136a24d 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -14,6 +14,7 @@ #include "application/app_state_manager.hpp" #include "blockchain/block_header_repository.hpp" #include "blockchain/block_tree.hpp" +#include "blockchain/block_tree_error.hpp" #include "common/visitor.hpp" #include "consensus/authority/authority_manager_error.hpp" #include "consensus/authority/authority_update_observer_error.hpp" @@ -285,7 +286,7 @@ namespace kagome::authority { root_->current_authorities->id); } else if (last_finalized_block.number == 0) { - auto& genesis_hash = block_tree_->getGenesisBlockHash(); + auto &genesis_hash = block_tree_->getGenesisBlockHash(); PREPARE_TRY(initial_authorities, grandpa_api_->authorities(genesis_hash), "Can't get grandpa authorities for genesis block: {}", @@ -628,8 +629,8 @@ namespace kagome::authority { return outcome::success(); }; - auto new_node = ancestor_node->makeDescendant( - {delay_start, delay_start_hash}, true); + auto new_node = + ancestor_node->makeDescendant({delay_start, delay_start_hash}, true); OUTCOME_TRY(force_change(new_node)); From 5f50a1509e4ed04ba8ea5d644e05c2297e51d1c9 Mon Sep 17 00:00:00 2001 From: Harrm Date: Sun, 17 Jul 2022 20:12:08 +0300 Subject: [PATCH 56/74] Fix authority manager tests' ASAN problem --- core/consensus/authority/impl/CMakeLists.txt | 1 + test/core/consensus/authority/authority_manager_test.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/consensus/authority/impl/CMakeLists.txt b/core/consensus/authority/impl/CMakeLists.txt index bba560c419..915a1418e3 100644 --- a/core/consensus/authority/impl/CMakeLists.txt +++ b/core/consensus/authority/impl/CMakeLists.txt @@ -11,4 +11,5 @@ target_link_libraries(authority_manager logger authority_manager_error authority_update_observer_error + block_tree_error ) diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index 945dbfbfa6..7640f7a151 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -91,7 +91,7 @@ class AuthorityManagerTest : public testing::Test { hasher, persistent_storage); - auto genesis_hash = "genesis"_hash256; + static auto genesis_hash = "genesis"_hash256; ON_CALL(*block_tree, getGenesisBlockHash()) .WillByDefault(testing::ReturnRef(genesis_hash)); From e3de787168a7e50321dd4e4071e77ab66fa5ff6e Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 15 Aug 2022 14:05:16 +0300 Subject: [PATCH 57/74] Fixing authority manager --- .github/workflows/test.yml | 6 +- cmake/print.cmake | 2 +- .../impl/vote_crypto_provider_impl.hpp | 2 +- core/host_api/impl/offchain_extension.cpp | 14 +---- .../binaryen/core_api_factory_impl.cpp | 2 +- .../binaryen/module/module_instance_impl.cpp | 7 +++ .../binaryen/module/module_instance_impl.hpp | 1 + .../runtime/common/runtime_instances_pool.cpp | 55 ++++--------------- .../runtime/common/runtime_instances_pool.hpp | 23 +++++--- core/runtime/module_instance.hpp | 38 +++++++++++++ core/runtime/wavm/CMakeLists.txt | 1 + core/runtime/wavm/core_api_factory_impl.cpp | 4 +- .../wavm/instance_environment_factory.hpp | 2 + .../wavm/intrinsics/intrinsic_functions.cpp | 16 +++--- .../wavm/intrinsics/intrinsic_functions.hpp | 7 ++- .../wavm/intrinsics/intrinsic_module.cpp | 6 ++ core/runtime/wavm/module.cpp | 6 +- core/runtime/wavm/module.hpp | 4 +- core/runtime/wavm/module_instance.cpp | 28 +++++----- core/runtime/wavm/module_instance.hpp | 17 +++--- .../wavm/wavm_internal_memory_provider.hpp | 2 + core/utils/kagome_db_editor.cpp | 28 +++++++--- housekeeping/ci_install_mold.sh | 11 ---- .../core/host_api/offchain_extension_test.cpp | 6 +- test/core/runtime/executor_test.cpp | 2 + .../cmake/clang-11-cpp17-toolchain.cmake | 1 + .../cmake/clang-14-cpp17-toolchain.cmake | 1 + .../cmake/cpp17-toolchain.cmake | 4 ++ .../core/runtime/module_instance_mock.hpp | 2 + 29 files changed, 161 insertions(+), 137 deletions(-) create mode 100644 core/runtime/wavm/intrinsics/intrinsic_module.cpp delete mode 100755 housekeeping/ci_install_mold.sh create mode 100644 test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake create mode 100644 test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake create mode 100644 test/external-project-test/cmake/cpp17-toolchain.cmake diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 940ea55986..8b7dc4a21d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,7 +52,7 @@ jobs: - name: "Linux: clang-11 UBSAN" run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-11_cxx17.cmake -DUBSAN=ON - name: "Linux: clang-11 External Project" - run: ./housekeeping/make_external_build.sh -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-11_cxx17.cmake + run: ./housekeeping/make_external_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/clang-11-cpp17-toolchain.cmake - name: "Linux: clang-14 UBSAN" run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-14_cxx17.cmake -DUBSAN=ON @@ -65,8 +65,6 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - name: "${{ matrix.options.name }}" run: "${{ matrix.options.run }}" @@ -100,8 +98,6 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - name: makeBuild env: BUILD_FINAL_TARGET: ctest_coverage diff --git a/cmake/print.cmake b/cmake/print.cmake index 7291735b1b..e03e2642b7 100644 --- a/cmake/print.cmake +++ b/cmake/print.cmake @@ -3,6 +3,6 @@ function(print) endfunction() function(fatal_error) - message(FATAL_ERROR "[${CMAKE_PROJECT_NAME}] ${ARGV}") + message(FATAL_ERROOR "[${CMAKE_PROJECT_NAME}] ${ARGV}") endfunction() diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.hpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.hpp index 1fff4d2b8b..f030ec05c1 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.hpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.hpp @@ -40,7 +40,7 @@ namespace kagome::consensus::grandpa { const std::shared_ptr &keypair_; std::shared_ptr ed_provider_; - RoundNumber round_number_; + const RoundNumber round_number_; std::shared_ptr voter_set_; }; diff --git a/core/host_api/impl/offchain_extension.cpp b/core/host_api/impl/offchain_extension.cpp index 06c178fe11..3abec604fa 100644 --- a/core/host_api/impl/offchain_extension.cpp +++ b/core/host_api/impl/offchain_extension.cpp @@ -178,22 +178,12 @@ namespace kagome::host_api { auto [key_ptr, key_size] = runtime::PtrSize(key); auto key_buffer = memory.loadN(key_ptr, key_size); auto [expected_ptr, expected_size] = runtime::PtrSize(expected); - auto expected_encoded = memory.loadN(expected_ptr, expected_size); - auto expected_res = - scale::decode>(expected_encoded); - if (expected_res.has_error()) { - throw std::runtime_error("Invalid encoded data for expected arg"); - } - auto &expected_as_buffer{expected_res.value()}; - std::optional expected_as_view; - if (expected_as_buffer) { - expected_as_view.emplace(expected_as_buffer.value()); - } + auto expected_buffer = memory.loadN(expected_ptr, expected_size); auto [value_ptr, value_size] = runtime::PtrSize(value); auto value_buffer = memory.loadN(value_ptr, value_size); auto result = worker->localStorageCompareAndSet( - storage_type, key_buffer, expected_as_view, value_buffer); + storage_type, key_buffer, expected_buffer, value_buffer); return result; } diff --git a/core/runtime/binaryen/core_api_factory_impl.cpp b/core/runtime/binaryen/core_api_factory_impl.cpp index 617a7c597f..b3cce84b03 100644 --- a/core/runtime/binaryen/core_api_factory_impl.cpp +++ b/core/runtime/binaryen/core_api_factory_impl.cpp @@ -37,7 +37,7 @@ namespace kagome::runtime::binaryen { } private: - std::shared_ptr instance_; + std::shared_ptr instance_; std::shared_ptr env_factory_; const std::vector &code_; }; diff --git a/core/runtime/binaryen/module/module_instance_impl.cpp b/core/runtime/binaryen/module/module_instance_impl.cpp index 3feb1d6341..cab08a2270 100644 --- a/core/runtime/binaryen/module/module_instance_impl.cpp +++ b/core/runtime/binaryen/module/module_instance_impl.cpp @@ -161,4 +161,11 @@ namespace kagome::runtime::binaryen { segment.data.size())); } } + void ModuleInstanceImpl::borrow( + BorrowedInstance::PoolReleaseFunction release) { + // Releaser for OCWs - doesn't need a valid pointer + static thread_local ModuleInstance::BorrowedInstance pool_release_token{ + nullptr, release}; + } + } // namespace kagome::runtime::binaryen diff --git a/core/runtime/binaryen/module/module_instance_impl.hpp b/core/runtime/binaryen/module/module_instance_impl.hpp index 24259aa76b..db0b45b059 100644 --- a/core/runtime/binaryen/module/module_instance_impl.hpp +++ b/core/runtime/binaryen/module/module_instance_impl.hpp @@ -41,6 +41,7 @@ namespace kagome::runtime::binaryen { InstanceEnvironment const &getEnvironment() const override; outcome::result resetEnvironment() override; + void borrow(BorrowedInstance::PoolReleaseFunction release) override; void forDataSegment(DataSegmentProcessor const &callback) const override; diff --git a/core/runtime/common/runtime_instances_pool.cpp b/core/runtime/common/runtime_instances_pool.cpp index e59153a763..c4bed1ceb5 100644 --- a/core/runtime/common/runtime_instances_pool.cpp +++ b/core/runtime/common/runtime_instances_pool.cpp @@ -15,53 +15,14 @@ #include "runtime/runtime_upgrade_tracker.hpp" namespace kagome::runtime { - /** - * @brief Wrapper type over sptr. Allows to return instance - * back to the ModuleInstancePool upon destruction of - * BorrowedInstance. - */ - class BorrowedInstance : public ModuleInstance { - public: - BorrowedInstance(std::weak_ptr pool, - const RuntimeInstancesPool::RootHash &state, - std::shared_ptr instance) - : pool_{std::move(pool)}, - state_{state}, - instance_{std::move(instance)} {} - ~BorrowedInstance() { - if (auto pool = pool_.lock()) { - pool->release(state_, std::move(instance_)); - } - } - - outcome::result callExportFunction( - std::string_view name, common::BufferView encoded_args) const override { - return instance_->callExportFunction(name, encoded_args); - } - outcome::result> getGlobal( - std::string_view name) const override { - return instance_->getGlobal(name); - } - void forDataSegment(DataSegmentProcessor const &callback) const override { - return instance_->forDataSegment(callback); - } - InstanceEnvironment const &getEnvironment() const override { - return instance_->getEnvironment(); - } - outcome::result resetEnvironment() override { - return instance_->resetEnvironment(); - } - - private: - std::weak_ptr pool_; - RuntimeInstancesPool::RootHash state_; - std::shared_ptr instance_; - }; + using kagome::primitives::ThreadNumber; + using soralog::util::getThreadNumber; outcome::result> RuntimeInstancesPool::tryAcquire( const RuntimeInstancesPool::RootHash &state) { std::scoped_lock guard{mt_}; + auto tid = getThreadNumber(); auto &pool = pools_[state]; if (not pool.empty()) { @@ -81,12 +42,16 @@ namespace kagome::runtime { } void RuntimeInstancesPool::release( - const RuntimeInstancesPool::RootHash &state, - std::shared_ptr &&instance) { + const RuntimeInstancesPool::RootHash &state) { std::lock_guard guard{mt_}; + auto tid = getThreadNumber(); auto &pool = pools_[state]; - pool.emplace(std::move(instance)); + // if used instance found, release + auto node = pool.extract(tid); + BOOST_ASSERT(node); + node.key() = POOL_FREE_INSTANCE_ID; + pool.insert(std::move(node)); } std::optional> RuntimeInstancesPool::getModule( const RuntimeInstancesPool::RootHash &state) { diff --git a/core/runtime/common/runtime_instances_pool.hpp b/core/runtime/common/runtime_instances_pool.hpp index 1ec7fad9af..3e19f65a8b 100644 --- a/core/runtime/common/runtime_instances_pool.hpp +++ b/core/runtime/common/runtime_instances_pool.hpp @@ -8,9 +8,15 @@ #include "runtime/module_repository.hpp" -#include +#include +#include + +#include "log/logger.hpp" +#include "runtime/instance_environment.hpp" namespace kagome::runtime { + using kagome::primitives::ThreadNumber; + /** * LRU cache designed for small amounts of data (as its get() is O(N)) */ @@ -85,16 +91,18 @@ namespace kagome::runtime { */ class RuntimeInstancesPool final : public std::enable_shared_from_this { - using ModuleInstancePool = std::stack>; + using RootHash = storage::trie::RootHash; + using ModuleInstancePool = + std::multimap>; public: - using RootHash = storage::trie::RootHash; using ModuleCache = SmallLruCache>; /** - * @brief Instantiate new or reuse existing ModuleInstance for the provided - * state. + * @brief @brief Attempt to acquire a ModuleInstance for the provided state. + * If none available, instantiate a new one. If already acquired by this + * thread, return the ptr to the acquired instance. * * @param state - the merkle trie root of the state containing the code of * the runtime module we are acquiring an instance of. @@ -108,10 +116,8 @@ namespace kagome::runtime { * * @param state - the merkle trie root of the state containing the runtime * module code we are releasing an instance of. - * @param instance - instance to be released. */ - void release(const RootHash &state, - std::shared_ptr &&instance); + void release(const RootHash &state); /** * @brief Get the module for state from internal cache @@ -132,6 +138,7 @@ namespace kagome::runtime { private: std::mutex mt_; static constexpr size_t MODULES_CACHE_SIZE = 2; + static constexpr size_t POOL_FREE_INSTANCE_ID = 0; ModuleCache modules_{MODULES_CACHE_SIZE}; std::map pools_; }; diff --git a/core/runtime/module_instance.hpp b/core/runtime/module_instance.hpp index 0f96781ab7..485bd9e956 100644 --- a/core/runtime/module_instance.hpp +++ b/core/runtime/module_instance.hpp @@ -31,6 +31,36 @@ namespace kagome::runtime { public: virtual ~ModuleInstance() = default; + /** + * @brief Wrapper type over sptr. Allows to return instance + * back to the ModuleInstancePool upon destruction of + * BorrowedInstance. + */ + class BorrowedInstance { + public: + using PoolReleaseFunction = std::function; + + BorrowedInstance(std::shared_ptr instance, + PoolReleaseFunction cache_release = {}) + : instance_{std::move(instance)}, + cache_release_{std::move(cache_release)} {} + ~BorrowedInstance() { + if (cache_release_) { + cache_release_(); + } + } + bool operator==(std::nullptr_t) { + return instance_ == nullptr; + } + ModuleInstance *operator->() { + return instance_.get(); + } + + private: + std::shared_ptr instance_; + PoolReleaseFunction cache_release_; + }; + /** * Call the instance's function * @param name - name of the function @@ -52,6 +82,14 @@ namespace kagome::runtime { virtual InstanceEnvironment const &getEnvironment() const = 0; virtual outcome::result resetEnvironment() = 0; + + /** + * @brief Make thread borrow a wrapped pointer to this instance with custom + * deleter 'release' + * + * @param release - a deleter, that should be called upon thread destruction + */ + virtual void borrow(BorrowedInstance::PoolReleaseFunction release) = 0; }; } // namespace kagome::runtime diff --git a/core/runtime/wavm/CMakeLists.txt b/core/runtime/wavm/CMakeLists.txt index 4a8fef7d31..4a2effce35 100644 --- a/core/runtime/wavm/CMakeLists.txt +++ b/core/runtime/wavm/CMakeLists.txt @@ -21,6 +21,7 @@ kagome_install(wavm_memory) add_library(runtime_wavm core_api_factory_impl.cpp intrinsics/intrinsic_functions.cpp + intrinsics/intrinsic_module.cpp intrinsics/intrinsic_module_instance.cpp intrinsics/intrinsic_resolver_impl.cpp instance_environment_factory.cpp diff --git a/core/runtime/wavm/core_api_factory_impl.cpp b/core/runtime/wavm/core_api_factory_impl.cpp index b4aae0d842..0a48762038 100644 --- a/core/runtime/wavm/core_api_factory_impl.cpp +++ b/core/runtime/wavm/core_api_factory_impl.cpp @@ -43,7 +43,7 @@ namespace kagome::runtime::wavm { BOOST_ASSERT(last_compiled_module_); } - outcome::result> getInstanceAt( + outcome::result> getInstanceAt( std::shared_ptr, const primitives::BlockInfo &, const primitives::BlockHeader &) override { @@ -61,7 +61,7 @@ namespace kagome::runtime::wavm { } private: - std::shared_ptr instance_; + std::shared_ptr instance_; std::shared_ptr instance_env_factory_; std::shared_ptr compartment_; std::shared_ptr module_params_; diff --git a/core/runtime/wavm/instance_environment_factory.hpp b/core/runtime/wavm/instance_environment_factory.hpp index 628859c82f..a0b1033fb0 100644 --- a/core/runtime/wavm/instance_environment_factory.hpp +++ b/core/runtime/wavm/instance_environment_factory.hpp @@ -34,6 +34,8 @@ namespace kagome::runtime { } namespace kagome::runtime::wavm { + + class ModuleInstance; class IntrinsicModule; class IntrinsicModuleInstance; class IntrinsicResolver; diff --git a/core/runtime/wavm/intrinsics/intrinsic_functions.cpp b/core/runtime/wavm/intrinsics/intrinsic_functions.cpp index e07a891696..32dfdc24d3 100644 --- a/core/runtime/wavm/intrinsics/intrinsic_functions.cpp +++ b/core/runtime/wavm/intrinsics/intrinsic_functions.cpp @@ -12,26 +12,24 @@ namespace kagome::runtime::wavm { log::Logger logger; - static thread_local std::stack> + static thread_local std::stack< + std::shared_ptr> global_instances; void pushBorrowedRuntimeInstance( - std::shared_ptr borrowed_runtime_instance) { + std::shared_ptr + borrowed_runtime_instance) { global_instances.emplace(std::move(borrowed_runtime_instance)); } - void popBorrowedRuntimeInstance() { - BOOST_ASSERT(!global_instances.empty()); - global_instances.pop(); - } - - std::shared_ptr peekBorrowedRuntimeInstance() { + std::shared_ptr + peekBorrowedRuntimeInstance() { BOOST_ASSERT(!global_instances.empty()); return global_instances.top(); } std::shared_ptr peekHostApi() { - return peekBorrowedRuntimeInstance()->getEnvironment().host_api; + return (*peekBorrowedRuntimeInstance())->getEnvironment().host_api; } #undef WAVM_DEFINE_INTRINSIC_FUNCTION diff --git a/core/runtime/wavm/intrinsics/intrinsic_functions.hpp b/core/runtime/wavm/intrinsics/intrinsic_functions.hpp index 8cbfd05e96..e9f58d8a70 100644 --- a/core/runtime/wavm/intrinsics/intrinsic_functions.hpp +++ b/core/runtime/wavm/intrinsics/intrinsic_functions.hpp @@ -26,9 +26,10 @@ namespace kagome::runtime::wavm { std::shared_ptr peekHostApi(); void pushBorrowedRuntimeInstance( - std::shared_ptr borrowed_runtime_instance); - void popBorrowedRuntimeInstance(); - std::shared_ptr peekBorrowedRuntimeInstance(); + std::shared_ptr + borrowed_runtime_instance); + std::shared_ptr + peekBorrowedRuntimeInstance(); extern log::Logger logger; diff --git a/core/runtime/wavm/intrinsics/intrinsic_module.cpp b/core/runtime/wavm/intrinsics/intrinsic_module.cpp new file mode 100644 index 0000000000..06df20d9bd --- /dev/null +++ b/core/runtime/wavm/intrinsics/intrinsic_module.cpp @@ -0,0 +1,6 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/wavm/intrinsics/intrinsic_module.hpp" diff --git a/core/runtime/wavm/module.cpp b/core/runtime/wavm/module.cpp index a8aab8b758..78ad40c64e 100644 --- a/core/runtime/wavm/module.cpp +++ b/core/runtime/wavm/module.cpp @@ -72,8 +72,8 @@ namespace kagome::runtime::wavm { BOOST_ASSERT(module_); } - outcome::result> ModuleImpl::instantiate() - const { + outcome::result> + ModuleImpl::instantiate() const { const auto &ir_module = WAVM::Runtime::getModuleIR(module_); bool imports_memory = std::find_if(ir_module.imports.cbegin(), @@ -102,7 +102,7 @@ namespace kagome::runtime::wavm { auto env = env_factory_->make( memory_origin, internal_instance, new_intrinsic_module_instance); - auto instance = std::make_shared( + auto instance = std::make_shared( std::move(env), internal_instance, module_, compartment_); return instance; diff --git a/core/runtime/wavm/module.hpp b/core/runtime/wavm/module.hpp index 71eccb55b4..4a8b9e3eb3 100644 --- a/core/runtime/wavm/module.hpp +++ b/core/runtime/wavm/module.hpp @@ -36,8 +36,8 @@ namespace kagome::runtime::wavm { std::shared_ptr env_factory, gsl::span code); - outcome::result> instantiate() - const override; + outcome::result> + instantiate() const override; private: ModuleImpl(std::shared_ptr compartment, diff --git a/core/runtime/wavm/module_instance.cpp b/core/runtime/wavm/module_instance.cpp index ef91035de9..3125102626 100644 --- a/core/runtime/wavm/module_instance.cpp +++ b/core/runtime/wavm/module_instance.cpp @@ -62,10 +62,8 @@ static WAVM::Uptr getIndexValue(const WAVM::IR::Value &value, }; } -OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wavm, - ModuleInstanceImpl::Error, - e) { - using E = kagome::runtime::wavm::ModuleInstanceImpl::Error; +OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wavm, ModuleInstance::Error, e) { + using E = kagome::runtime::wavm::ModuleInstance::Error; switch (e) { case E::WRONG_ARG_COUNT: return "The provided function argument count should equal to 2"; @@ -82,7 +80,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wavm, namespace kagome::runtime::wavm { - ModuleInstanceImpl::ModuleInstanceImpl( + ModuleInstance::ModuleInstance( InstanceEnvironment &&env, WAVM::Runtime::GCPointer instance, WAVM::Runtime::ModuleRef module, @@ -97,7 +95,7 @@ namespace kagome::runtime::wavm { BOOST_ASSERT(module_ != nullptr); } - outcome::result ModuleInstanceImpl::callExportFunction( + outcome::result ModuleInstance::callExportFunction( std::string_view name, common::BufferView encoded_args) const { auto memory = env_.memory_provider->getCurrentMemory().value(); @@ -134,9 +132,6 @@ namespace kagome::runtime::wavm { // Allocate an array to receive the invocation results. BOOST_ASSERT(invokeSig.results().size() == 1); std::array untaggedInvokeResults; - pushBorrowedRuntimeInstance( - std::const_pointer_cast(shared_from_this())); - const auto pop = gsl::finally(&popBorrowedRuntimeInstance); try { WAVM::Runtime::unwindSignalsAsExceptions( [&context, @@ -165,7 +160,7 @@ namespace kagome::runtime::wavm { return res; } - outcome::result> ModuleInstanceImpl::getGlobal( + outcome::result> ModuleInstance::getGlobal( std::string_view name) const { auto global = WAVM::Runtime::asGlobalNullable( WAVM::Runtime::getInstanceExport(instance_, name.data())); @@ -190,7 +185,7 @@ namespace kagome::runtime::wavm { } } - void ModuleInstanceImpl::forDataSegment( + void ModuleInstance::forDataSegment( DataSegmentProcessor const &callback) const { using WAVM::Uptr; using WAVM::IR::DataSegment; @@ -212,12 +207,19 @@ namespace kagome::runtime::wavm { } } - InstanceEnvironment const &ModuleInstanceImpl::getEnvironment() const { + InstanceEnvironment const &ModuleInstance::getEnvironment() const { return env_; } - outcome::result ModuleInstanceImpl::resetEnvironment() { + outcome::result ModuleInstance::resetEnvironment() { env_.host_api->reset(); return outcome::success(); } + + void ModuleInstance::borrow(BorrowedInstance::PoolReleaseFunction release) { + auto borrowed = std::make_shared( + shared_from_this(), release); + pushBorrowedRuntimeInstance(std::move(borrowed)); + } + } // namespace kagome::runtime::wavm diff --git a/core/runtime/wavm/module_instance.hpp b/core/runtime/wavm/module_instance.hpp index fe8449d5a6..17a16e6760 100644 --- a/core/runtime/wavm/module_instance.hpp +++ b/core/runtime/wavm/module_instance.hpp @@ -26,9 +26,8 @@ namespace kagome::runtime::wavm { class CompartmentWrapper; - class ModuleInstanceImpl - : public ModuleInstance, - public std::enable_shared_from_this { + class ModuleInstance : public runtime::ModuleInstance, + public std::enable_shared_from_this { public: enum class Error { FUNC_NOT_FOUND = 1, @@ -36,11 +35,10 @@ namespace kagome::runtime::wavm { EXECUTION_ERROR, WRONG_RETURN_TYPE }; - ModuleInstanceImpl( - InstanceEnvironment &&env, - WAVM::Runtime::GCPointer instance, - WAVM::Runtime::ModuleRef module, - std::shared_ptr compartment); + ModuleInstance(InstanceEnvironment &&env, + WAVM::Runtime::GCPointer instance, + WAVM::Runtime::ModuleRef module, + std::shared_ptr compartment); outcome::result callExportFunction( std::string_view name, common::BufferView encoded_args) const override; @@ -52,6 +50,7 @@ namespace kagome::runtime::wavm { InstanceEnvironment const &getEnvironment() const override; outcome::result resetEnvironment() override; + void borrow(BorrowedInstance::PoolReleaseFunction release) override; private: InstanceEnvironment env_; @@ -63,6 +62,6 @@ namespace kagome::runtime::wavm { } // namespace kagome::runtime::wavm -OUTCOME_HPP_DECLARE_ERROR(kagome::runtime::wavm, ModuleInstanceImpl::Error) +OUTCOME_HPP_DECLARE_ERROR(kagome::runtime::wavm, ModuleInstance::Error) #endif // KAGOME_CORE_RUNTIME_WAVM_IMPL_MODULE_INSTANCE_HPP diff --git a/core/runtime/wavm/wavm_internal_memory_provider.hpp b/core/runtime/wavm/wavm_internal_memory_provider.hpp index 2e9ca8a536..4846d94cda 100644 --- a/core/runtime/wavm/wavm_internal_memory_provider.hpp +++ b/core/runtime/wavm/wavm_internal_memory_provider.hpp @@ -13,6 +13,8 @@ namespace WAVM::Runtime { } namespace kagome::runtime::wavm { + + class ModuleInstance; class MemoryImpl; class WavmInternalMemoryProvider final : public MemoryProvider { diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index cbbfe97960..67051e1696 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -39,8 +39,10 @@ using sptr = std::shared_ptr; template struct is_optional : std::false_type {}; + template struct is_optional> : std::true_type {}; + template inline auto check(T &&res) { if (not res.has_value()) { @@ -169,14 +171,15 @@ void child_storage_root_hashes( } } +auto is_hash(const char *s) { + return std::strlen(s) == common::Hash256::size() * 2 + 2 + && std::equal(s, s + 2, "0x"); +}; + int main(int argc, char *argv[]) { backward::SignalHandling sh; Command cmd; - auto is_hash = [](const char *s) { - return std::strlen(s) == common::Hash256::size() + 2 - && std::equal(s, s + 2, "0x"); - }; if (argc == 2 or (argc == 3 && is_hash(argv[2])) or (argc == 4 and std::strcmp(argv[MODE], "compact") == 0)) { cmd = COMPACT; @@ -186,6 +189,15 @@ int main(int argc, char *argv[]) { usage(); return 0; } + std::optional target_state_param; + if (argc > 2) { + if (!is_hash(argv[2])) { + std::cout << "ERROR: Invalid state hash\n"; + usage(); + return -1; + } + target_state_param = RootHash::fromHexWithPrefix(argv[2]).value(); + } auto logging_system = std::make_shared( std::make_shared()); @@ -298,6 +310,8 @@ int main(int argc, char *argv[]) { leafs.emplace(header.number - 1, header.parent_hash); to_remove.insert(std::move(node)); } + RootHash target_state = + target_state_param.value_or(last_finalized_block_state_root); log->trace("Autodetected finalized block is {}, state root is {:l}", last_finalized_block, @@ -333,11 +347,9 @@ int main(int argc, char *argv[]) { .value(); if (COMPACT == cmd) { - auto batch = - check(persistent_batch(trie, last_finalized_block_state_root)) - .value(); + auto batch = check(persistent_batch(trie, target_state)).value(); auto finalized_batch = - check(persistent_batch(trie, last_finalized_block_state_root)) + check(persistent_batch(trie, target_state)) .value(); std::vector> child_batches; diff --git a/housekeeping/ci_install_mold.sh b/housekeeping/ci_install_mold.sh deleted file mode 100755 index c7f7d0c567..0000000000 --- a/housekeeping/ci_install_mold.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -xe - -if [ "$CI" != "true" ] || [ "$GITHUB_ACTIONS" != "true" ]; then - echo "CI=true GITHUB_ACTIONS=true are required" - exit -1 -fi - -wget -q -O- https://github.com/rui314/mold/releases/download/v1.3.1/mold-1.3.1-x86_64-linux.tar.gz | tar -C /usr/local --strip-components=1 -xzf - -if [ "$1" == "--make-default" ]; then - ln -sf /usr/local/bin/mold $(realpath /usr/bin/ld) -fi diff --git a/test/core/host_api/offchain_extension_test.cpp b/test/core/host_api/offchain_extension_test.cpp index 59f4443c27..00776ae634 100644 --- a/test/core/host_api/offchain_extension_test.cpp +++ b/test/core/host_api/offchain_extension_test.cpp @@ -25,7 +25,6 @@ #include "testutil/prepare_loggers.hpp" using kagome::common::Buffer; -using kagome::common::BufferView; using kagome::host_api::OffchainExtension; using kagome::host_api::OffchainExtensionConfig; using kagome::offchain::Failure; @@ -299,15 +298,14 @@ TEST_P(TernaryParametrizedTest, LocalStorageCAS) { WasmPointer expected_pointer = 45; WasmSize expected_size = 45; WasmSpan expected_span = PtrSize(expected_pointer, expected_size).combine(); - auto expected = Buffer{scale::encode(std::optional{}).value()}; + Buffer expected(8, 'e'); EXPECT_CALL(*memory_, loadN(key_pointer, key_size)).WillOnce(Return(key)); EXPECT_CALL(*memory_, loadN(value_pointer, value_size)) .WillOnce(Return(value)); EXPECT_CALL(*memory_, loadN(expected_pointer, expected_size)) .WillOnce(Return(expected)); EXPECT_CALL(*offchain_worker_, - localStorageCompareAndSet( - _, key.view(), std::optional{}, value)) + localStorageCompareAndSet(_, key.view(), _, value)) .WillOnce(Return(true)); offchain_extension_->ext_offchain_local_storage_compare_and_set_version_1( GetParam(), key_span, expected_span, value_span); diff --git a/test/core/runtime/executor_test.cpp b/test/core/runtime/executor_test.cpp index 559a793165..ee65aa0989 100644 --- a/test/core/runtime/executor_test.cpp +++ b/test/core/runtime/executor_test.cpp @@ -74,6 +74,7 @@ class ExecutorTest : public testing::Test { kagome::storage::trie::RootHash const &next_storage_state) { static Buffer enc_args; enc_args = Buffer{scale::encode(arg1, arg2).value()}; + const PtrSize ARGS_LOCATION{1, 2}; const PtrSize RESULT_LOCATION{3, 4}; Buffer enc_res{scale::encode(res).value()}; @@ -143,6 +144,7 @@ class ExecutorTest : public testing::Test { int res) { static Buffer enc_args; enc_args = Buffer{scale::encode(arg1, arg2).value()}; + const PtrSize ARGS_LOCATION{1, 2}; const PtrSize RESULT_LOCATION{3, 4}; Buffer enc_res{scale::encode(res).value()}; EXPECT_CALL(*memory_, loadN(RESULT_LOCATION.ptr, RESULT_LOCATION.size)) diff --git a/test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake b/test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake new file mode 100644 index 0000000000..536e4a008d --- /dev/null +++ b/test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake @@ -0,0 +1 @@ +include("${CMAKE_CURRENT_LIST_DIR}/cpp17-toolchain.cmake") \ No newline at end of file diff --git a/test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake b/test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake new file mode 100644 index 0000000000..536e4a008d --- /dev/null +++ b/test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake @@ -0,0 +1 @@ +include("${CMAKE_CURRENT_LIST_DIR}/cpp17-toolchain.cmake") \ No newline at end of file diff --git a/test/external-project-test/cmake/cpp17-toolchain.cmake b/test/external-project-test/cmake/cpp17-toolchain.cmake new file mode 100644 index 0000000000..5c08ad5d6c --- /dev/null +++ b/test/external-project-test/cmake/cpp17-toolchain.cmake @@ -0,0 +1,4 @@ + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/test/mock/core/runtime/module_instance_mock.hpp b/test/mock/core/runtime/module_instance_mock.hpp index 0134dd0388..b8e9ee3058 100644 --- a/test/mock/core/runtime/module_instance_mock.hpp +++ b/test/mock/core/runtime/module_instance_mock.hpp @@ -34,6 +34,8 @@ namespace kagome::runtime { (), (const, override)); + MOCK_METHOD(void, borrow, (std::function), (override)); + MOCK_METHOD(outcome::result, resetEnvironment, (), (override)); }; } // namespace kagome::runtime From 8d9b7ac26af030b2f4e6732a6f65db285621baf8 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 25 Aug 2022 15:06:39 +0300 Subject: [PATCH 58/74] Fix authority manager on synced chain --- core/blockchain/impl/block_storage_impl.cpp | 13 +- core/blockchain/impl/block_tree_impl.cpp | 2 +- .../consensus/authority/authority_manager.hpp | 15 + .../authority/impl/authority_manager_impl.cpp | 157 +++++------ .../authority/impl/authority_manager_impl.hpp | 5 + core/consensus/grandpa/impl/grandpa_impl.cpp | 8 +- .../impl/vote_crypto_provider_impl.cpp | 17 +- .../grandpa/impl/voting_round_error.cpp | 2 + .../grandpa/impl/voting_round_error.hpp | 1 + core/crypto/twox/CMakeLists.txt | 2 +- core/injector/application_injector.cpp | 39 ++- core/injector/application_injector.hpp | 10 + core/primitives/CMakeLists.txt | 1 + core/primitives/block_header.cpp | 11 + core/primitives/block_header.hpp | 7 + .../binaryen/core_api_factory_impl.cpp | 2 +- .../binaryen/module/module_instance_impl.cpp | 6 - .../binaryen/module/module_instance_impl.hpp | 1 - .../runtime/common/runtime_instances_pool.cpp | 57 +++- .../runtime/common/runtime_instances_pool.hpp | 265 +++++++++--------- core/runtime/module_instance.hpp | 38 --- core/runtime/wavm/core_api_factory_impl.cpp | 4 +- .../wavm/instance_environment_factory.hpp | 2 - .../wavm/intrinsics/intrinsic_functions.cpp | 16 +- .../wavm/intrinsics/intrinsic_functions.hpp | 7 +- core/runtime/wavm/module.cpp | 6 +- core/runtime/wavm/module.hpp | 4 +- core/runtime/wavm/module_instance.cpp | 28 +- core/runtime/wavm/module_instance.hpp | 17 +- .../wavm/wavm_internal_memory_provider.hpp | 2 - core/utils/CMakeLists.txt | 3 +- core/utils/storage_explorer.cpp | 109 +++++-- .../authority/authority_manager_test.cpp | 9 +- .../voting_round/voting_round_test.cpp | 9 +- test/mock/core/runtime/grandpa_api_mock.hpp | 5 + .../core/runtime/module_instance_mock.hpp | 2 - 36 files changed, 502 insertions(+), 380 deletions(-) create mode 100644 core/primitives/block_header.cpp diff --git a/core/blockchain/impl/block_storage_impl.cpp b/core/blockchain/impl/block_storage_impl.cpp index 2865993c3c..f63f15c385 100644 --- a/core/blockchain/impl/block_storage_impl.cpp +++ b/core/blockchain/impl/block_storage_impl.cpp @@ -188,9 +188,6 @@ namespace kagome::blockchain { const primitives::BlockDataFlags &remove_flags) { primitives::BlockData to_insert; - // if block data does not exist, put a new one. Otherwise, get the old one - // and merge with the new one. During the merge new block data fields have - // higher priority over the old ones (old ones should be rewritten) OUTCOME_TRY(existing_block_data_opt, getBlockData(remove_flags.hash)); if (not existing_block_data_opt.has_value()) { return outcome::success(); @@ -207,15 +204,15 @@ namespace kagome::blockchain { // add all the fields from the new block_data to_insert.header = - move_if_flag(remove_flags.header, std::move(existing_data.header)); + move_if_flag(!remove_flags.header, std::move(existing_data.header)); to_insert.body = - move_if_flag(remove_flags.body, std::move(existing_data.body)); + move_if_flag(!remove_flags.body, std::move(existing_data.body)); to_insert.justification = move_if_flag( - remove_flags.justification, std::move(existing_data.justification)); + !remove_flags.justification, std::move(existing_data.justification)); to_insert.message_queue = move_if_flag( - remove_flags.message_queue, std::move(existing_data.message_queue)); + !remove_flags.message_queue, std::move(existing_data.message_queue)); to_insert.receipt = - move_if_flag(remove_flags.receipt, std::move(existing_data.receipt)); + move_if_flag(!remove_flags.receipt, std::move(existing_data.receipt)); OUTCOME_TRY(encoded_block_data, scale::encode(to_insert)); OUTCOME_TRY(putWithPrefix(*storage_, diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index ea39d36be6..f6eb3ca78c 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -874,7 +874,7 @@ namespace kagome::blockchain { storage_->getJustification(last_finalized_block_info.hash)); if (justification_opt.has_value()) { SL_DEBUG(log_, - "Purge redunant justification for finalized block {}", + "Purge redundant justification for finalized block {}", last_finalized_block_info); OUTCOME_TRY(storage_->removeJustification( last_finalized_block_info.hash, last_finalized_block_info.number)); diff --git a/core/consensus/authority/authority_manager.hpp b/core/consensus/authority/authority_manager.hpp index 787b1dd6be..2cec07f3a5 100644 --- a/core/consensus/authority/authority_manager.hpp +++ b/core/consensus/authority/authority_manager.hpp @@ -12,14 +12,29 @@ #include "primitives/authority.hpp" #include "primitives/block_header.hpp" +namespace kagome::storage::trie { + class TrieStorage; +} + +namespace kagome::crypto { + class Hasher; +} + namespace kagome::authority { using IsBlockFinalized = Tagged; + outcome::result> fetchSetIdFromTrieStorage( + storage::trie::TrieStorage const &trie_storage, + crypto::Hasher const &hasher, + storage::trie::RootHash const &state); + class AuthorityManager { public: virtual ~AuthorityManager() = default; + virtual outcome::result initializeAt(const primitives::BlockInfo& root_block) = 0; + /** * Recalculate the authority change graph starting from genesis and up to * the last finalized block. The result shall be stored in the provided diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 33d136a24d..a71cc124a9 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -103,8 +103,8 @@ namespace kagome::authority { outcome::result> fetchSetIdFromTrieStorage( storage::trie::TrieStorage const &trie_storage, crypto::Hasher const &hasher, - primitives::BlockHeader const &header) { - OUTCOME_TRY(batch, trie_storage.getEphemeralBatchAt(header.state_root)); + storage::trie::RootHash const &state) { + OUTCOME_TRY(batch, trie_storage.getEphemeralBatchAt(state)); std::optional set_id_opt; auto current_set_id_keypart = @@ -150,9 +150,6 @@ namespace kagome::authority { return outcome::success(); } - static const common::Buffer kForcedSetIdListKey = - common::Buffer::fromString(":authority_manager:forced_set_id_list"); - /** * Collect all consensus messages found in finalized block starting from * {@param finalized_block_hash} and until an authority set change is reached. @@ -247,39 +244,46 @@ namespace kagome::authority { BOOST_UNREACHABLE_RETURN({}) } -#define CONCAT(first, second) CONCAT_UTIL(first, second) -#define CONCAT_UTIL(first, second) first##second -#define UNIQUE_NAME(tag) CONCAT(tag, __LINE__) - -#define PREPARE_TRY_VOID(expr_res, error_msg, ...) \ - auto &&UNIQUE_NAME(expr_r_) = (expr_res); \ - if (UNIQUE_NAME(expr_r_).has_error()) { \ - auto &error = UNIQUE_NAME(expr_r_).error(); \ - log_->critical(error_msg, __VA_ARGS__); \ - return false; \ + bool AuthorityManagerImpl::prepare() { + const auto finalized_block = block_tree_->getLastFinalized(); + auto res = initializeAt(finalized_block); + if (!res) { + SL_ERROR(log_, + "Error initializing authority manager: {}", + res.error().message()); + } + return res.has_value(); } -#define PREPARE_TRY(val, expr_res, error_msg, ...) \ - PREPARE_TRY_VOID(expr_res, error_msg, __VA_ARGS__); \ - auto &val = UNIQUE_NAME(expr_r_).value(); + outcome::result AuthorityManagerImpl::initializeAt( + const primitives::BlockInfo &root_block) { + OUTCOME_TRY(collected_msgs, + collectMsgsFromNonFinalBlocks(*block_tree_, root_block.hash)); - bool AuthorityManagerImpl::prepare() { - const auto finalized_block = block_tree_->getLastFinalized(); + OUTCOME_TRY(graph_root_block, + collectConsensusMsgsUntilNearestSetChangeTo( + collected_msgs, root_block, *block_tree_, log_)); - PREPARE_TRY( - collected_msgs, - collectMsgsFromNonFinalBlocks(*block_tree_, finalized_block.hash), - "Error collecting consensus messages from non-finalized blocks: {}", - error.message()); + OUTCOME_TRY(root_header, + block_tree_->getBlockHeader(graph_root_block.hash)); - PREPARE_TRY(opt_root, - fetchScheduleGraphRoot(*persistent_storage_), - "Error fetching authority set from persistent storage: {}", - error.message()); + OUTCOME_TRY(set_id_from_runtime, readSetIdFromRuntime(root_header)); + + OUTCOME_TRY(opt_root, fetchScheduleGraphRoot(*persistent_storage_)); auto last_finalized_block = block_tree_->getLastFinalized(); if (opt_root && opt_root.value()->current_block.number <= last_finalized_block.number) { + // FIXME: Correction to bypass bug where after finishing syncing and + // restarting the node we get a set id off by one + if (opt_root.value()->current_authorities->id + == set_id_from_runtime - 1) { + auto &authority_list = + opt_root.value()->current_authorities->authorities; + opt_root.value()->current_authorities = + std::make_shared(set_id_from_runtime, + authority_list); + } root_ = std::move(opt_root.value()); SL_TRACE(log_, "Fetched authority set graph root from database with id {}", @@ -287,10 +291,7 @@ namespace kagome::authority { } else if (last_finalized_block.number == 0) { auto &genesis_hash = block_tree_->getGenesisBlockHash(); - PREPARE_TRY(initial_authorities, - grandpa_api_->authorities(genesis_hash), - "Can't get grandpa authorities for genesis block: {}", - error.message()); + OUTCOME_TRY(initial_authorities, grandpa_api_->authorities(genesis_hash)); root_ = authority::ScheduleNode::createAsRoot( std::make_shared( 0, std::move(initial_authorities)), @@ -301,44 +302,15 @@ namespace kagome::authority { "Storage does not contain valid info about the root authority set; " "Fall back to obtaining it from the runtime storage (which may " "fail after a forced authority change happened on chain)"); - PREPARE_TRY( - graph_root_block, - collectConsensusMsgsUntilNearestSetChangeTo( - collected_msgs, finalized_block, *block_tree_, log_), - "Error collecting consensus messages from finalized blocks: {}", - error.message()); - PREPARE_TRY(authorities, - grandpa_api_->authorities(graph_root_block.hash), - "Can't get grandpa authorities for block {}: {}", - graph_root_block, - error.message()); - - PREPARE_TRY(header, - block_tree_->getBlockHeader(graph_root_block.hash), - "Can't get header of block {}: {}", - graph_root_block, - error.message()); - AuthoritySetId set_id{}; - auto set_id_res = grandpa_api_->current_set_id(graph_root_block.hash); - if (set_id_res) { - set_id = set_id_res.value(); - } else { - PREPARE_TRY( - set_id_, - fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header), - "Failed to fetch authority set id from storage for block {}: {}", - graph_root_block, - error.message()); - set_id = set_id_.value(); - } + OUTCOME_TRY(authorities, + grandpa_api_->authorities(graph_root_block.hash)); + auto authority_set = std::make_shared( - set_id, std::move(authorities)); + set_id_from_runtime, std::move(authorities)); root_ = authority::ScheduleNode::createAsRoot(authority_set, graph_root_block); - PREPARE_TRY_VOID(storeScheduleGraphRoot(*persistent_storage_, *root_), - "Failed to store schedule graph root: {}", - error.message()); + OUTCOME_TRY(storeScheduleGraphRoot(*persistent_storage_, *root_)); SL_TRACE(log_, "Create authority set graph root with id {}, taken from runtime " "storage", @@ -347,18 +319,16 @@ namespace kagome::authority { while (not collected_msgs.empty()) { const auto &args = collected_msgs.top(); - PREPARE_TRY_VOID(onConsensus(args.block, args.message), - "Can't apply previous consensus message: {}", - error.message()); + OUTCOME_TRY(onConsensus(args.block, args.message)); collected_msgs.pop(); } // prune to reorganize collected changes - prune(finalized_block); + prune(root_block); SL_DEBUG(log_, - "Actual grandpa authority set (id={}):", + "Current grandpa authority set (id={}):", root_->current_authorities->id); size_t index = 0; for (const auto &authority : *root_->current_authorities) { @@ -370,11 +340,24 @@ namespace kagome::authority { authority.weight); } - return true; + return outcome::success(); } -#undef PREPARE_TRY_VOID -#undef PREPARE_TRY + outcome::result AuthorityManagerImpl::readSetIdFromRuntime( + primitives::BlockHeader const &header) const { + AuthoritySetId set_id{}; + OUTCOME_TRY(hash, primitives::calculateBlockHash(header, *hasher_)); + auto set_id_res = grandpa_api_->current_set_id(hash); + if (set_id_res) { + set_id = set_id_res.value(); + } else { + OUTCOME_TRY(set_id_, + fetchSetIdFromTrieStorage( + *trie_storage_, *hasher_, header.state_root)); + set_id = set_id_.value(); + } + return set_id; + } outcome::result AuthorityManagerImpl::recalculateStoredState( primitives::BlockNumber last_finalized_number) { @@ -398,10 +381,10 @@ namespace kagome::authority { auto start = std::chrono::steady_clock::now(); for (primitives::BlockNumber number = 0; number <= last_finalized_number; number++) { - auto start = std::chrono::steady_clock::now(); auto header_res = header_repo_->getBlockHeader(number); - if(!header_res) continue; // Temporary workaround about the justification pruning bug - auto& header = header_res.value(); + if (!header_res) + continue; // Temporary workaround about the justification pruning bug + auto &header = header_res.value(); OUTCOME_TRY(hash, header_repo_->getHashByNumber(number)); primitives::BlockInfo info{number, hash}; @@ -452,9 +435,6 @@ namespace kagome::authority { ? (bool)finalized : node->current_block.number <= block_tree_->getLastFinalized().number; - if (target_block.number == 1514496) { - node_in_finalized_chain = true; - } auto adjusted_node = node->makeDescendant(target_block, node_in_finalized_chain); @@ -468,7 +448,7 @@ namespace kagome::authority { // Since getAppropriateAncestor worked normally on this block auto header = block_tree_->getBlockHeader(target_block.hash).value(); auto id_from_storage = - fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header) + fetchSetIdFromTrieStorage(*trie_storage_, *hasher_, header.state_root) .value() .value(); SL_DEBUG( @@ -577,14 +557,13 @@ namespace kagome::authority { delay, current_block, delay_start + delay); - auto delay_start_header_res = - block_tree_->getBlockHeader(delay_start + 1); + auto delay_start_header_res = block_tree_->getBlockHeader(delay_start + 1); if (delay_start_header_res.has_error()) { SL_ERROR(log_, "Failed to obtain header by number {}", delay_start + 1); } OUTCOME_TRY(delay_start_header, delay_start_header_res); auto ancestor_node = - getAppropriateAncestor({delay_start, delay_start_hash}); + getAppropriateAncestor({delay_start, delay_start_header.parent_hash}); if (not ancestor_node) { return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; @@ -629,8 +608,8 @@ namespace kagome::authority { return outcome::success(); }; - auto new_node = - ancestor_node->makeDescendant({delay_start, delay_start_hash}, true); + auto new_node = ancestor_node->makeDescendant( + {delay_start, delay_start_header.parent_hash}, true); OUTCOME_TRY(force_change(new_node)); @@ -802,7 +781,9 @@ namespace kagome::authority { [](auto &) { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; }); - } else if (message.consensus_engine_id == primitives::kBabeEngineId) { + } else if (message.consensus_engine_id == primitives::kBabeEngineId + || message.consensus_engine_id + == primitives::kUnsupportedEngineId_BEEF) { // ignore return outcome::success(); diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 51e9caf4f9..07e6861b4d 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -70,6 +70,8 @@ namespace kagome::authority { bool prepare(); + outcome::result initializeAt(const primitives::BlockInfo &root_block); + primitives::BlockInfo base() const override; std::optional> authorities( @@ -115,6 +117,9 @@ namespace kagome::authority { std::shared_ptr getAppropriateAncestor( const primitives::BlockInfo &block) const; + outcome::result readSetIdFromRuntime( + primitives::BlockHeader const &targetBlock) const; + /** * @brief Check if one block is direct ancestor of second one * @param ancestor - hash of block, which is at the top of the chain diff --git a/core/consensus/grandpa/impl/grandpa_impl.cpp b/core/consensus/grandpa/impl/grandpa_impl.cpp index 8de9e686f2..3d0f2a591f 100644 --- a/core/consensus/grandpa/impl/grandpa_impl.cpp +++ b/core/consensus/grandpa/impl/grandpa_impl.cpp @@ -849,10 +849,12 @@ namespace kagome::consensus::grandpa { auto &authority_set = authorities_opt.value(); // This is justification for non-actual round - if (authority_set->id < current_round_->voterSetId() - or (authority_set->id == current_round_->voterSetId() + if (authority_set->id < current_round_->voterSetId()) { + return VotingRoundError::JUSTIFICATION_FOR_AUTHORITY_SET_IN_PAST; + } + if (authority_set->id == current_round_->voterSetId() && justification.round_number - < current_round_->roundNumber())) { + < current_round_->roundNumber()) { return VotingRoundError::JUSTIFICATION_FOR_ROUND_IN_PAST; } diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index 88baf3c21f..47da1c9a8b 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -37,7 +37,22 @@ namespace kagome::consensus::grandpa { scale::encode(vote.message, number, voter_set_->id()).value(); auto verifying_result = ed_provider_->verify(vote.signature, payload, vote.id); - return verifying_result.has_value() and verifying_result.value(); + bool result = verifying_result.has_value() and verifying_result.value(); + if (!result) { + auto logger = log::createLogger("VoteCryptoProvider", "authority"); + for (auto id = voter_set_->id() - 50; id < voter_set_->id() + 50; id++) { + auto payload = + scale::encode(vote.message, number, id).value(); + auto verifying_result = + ed_provider_->verify(vote.signature, payload, vote.id); + if (verifying_result.has_value() and verifying_result.value()) { + SL_DEBUG(logger, "Correct set id is {}, actual is {}", id, voter_set_->id()); + return false; + } + } + SL_DEBUG(logger, "Failed to find correct set id"); + } + return result; } bool VoteCryptoProviderImpl::verifyPrimaryPropose( diff --git a/core/consensus/grandpa/impl/voting_round_error.cpp b/core/consensus/grandpa/impl/voting_round_error.cpp index 27ac9200b7..9d475f4a68 100644 --- a/core/consensus/grandpa/impl/voting_round_error.cpp +++ b/core/consensus/grandpa/impl/voting_round_error.cpp @@ -18,6 +18,8 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::grandpa, VotingRoundError, e) { return "Justification for passed round"; case E::JUSTIFICATION_FOR_BLOCK_IN_PAST: return "Justification for early finalized block"; + case E::JUSTIFICATION_FOR_AUTHORITY_SET_IN_PAST: + return "Justification for an obsolete authority set"; case E::LAST_ESTIMATE_BETTER_THAN_PREVOTE: return "Current state does not contain prevote which is equal to the " "last round estimate or is descendant of it"; diff --git a/core/consensus/grandpa/impl/voting_round_error.hpp b/core/consensus/grandpa/impl/voting_round_error.hpp index 623be9d023..b45780b0a6 100644 --- a/core/consensus/grandpa/impl/voting_round_error.hpp +++ b/core/consensus/grandpa/impl/voting_round_error.hpp @@ -16,6 +16,7 @@ namespace kagome::consensus::grandpa { NOT_ENOUGH_WEIGHT, JUSTIFICATION_FOR_ROUND_IN_PAST, JUSTIFICATION_FOR_BLOCK_IN_PAST, + JUSTIFICATION_FOR_AUTHORITY_SET_IN_PAST, JUSTIFIED_BLOCK_IS_GREATER_THAN_ACTUALLY_FINALIZED, NO_KNOWN_AUTHORITIES_FOR_BLOCK, WRONG_ORDER_OF_VOTER_SET_ID, diff --git a/core/crypto/twox/CMakeLists.txt b/core/crypto/twox/CMakeLists.txt index 8f3cb2d8c2..55474e16a9 100644 --- a/core/crypto/twox/CMakeLists.txt +++ b/core/crypto/twox/CMakeLists.txt @@ -5,7 +5,7 @@ add_library(twox twox.cpp) target_link_libraries(twox - xxhash + xxhash blob ) kagome_install(twox) diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 52f8220b2e..477859b8d1 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -13,12 +13,24 @@ #include #include #include -#undef U64 // comes from OpenSSL and messes with WAVM #include #include #include #include +#undef U64 // comes from OpenSSL and messes with WAVM + +#include "runtime/wavm/compartment_wrapper.hpp" +#include "runtime/wavm/core_api_factory_impl.hpp" +#include "runtime/wavm/instance_environment_factory.hpp" +#include "runtime/wavm/intrinsics/intrinsic_functions.hpp" +#include "runtime/wavm/intrinsics/intrinsic_module.hpp" +#include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" +#include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" +#include "runtime/wavm/module.hpp" +#include "runtime/wavm/module_cache.hpp" +#include "runtime/wavm/module_factory_impl.hpp" + #include "api/service/author/author_jrpc_processor.hpp" #include "api/service/author/impl/author_api_impl.hpp" #include "api/service/chain/chain_jrpc_processor.hpp" @@ -128,16 +140,6 @@ #include "runtime/runtime_api/impl/session_keys_api.hpp" #include "runtime/runtime_api/impl/tagged_transaction_queue.hpp" #include "runtime/runtime_api/impl/transaction_payment_api.hpp" -#include "runtime/wavm/compartment_wrapper.hpp" -#include "runtime/wavm/core_api_factory_impl.hpp" -#include "runtime/wavm/instance_environment_factory.hpp" -#include "runtime/wavm/intrinsics/intrinsic_functions.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" -#include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" -#include "runtime/wavm/module.hpp" -#include "runtime/wavm/module_cache.hpp" -#include "runtime/wavm/module_factory_impl.hpp" #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp" #include "storage/predefined_keys.hpp" #include "storage/rocksdb/rocksdb.hpp" @@ -1614,4 +1616,19 @@ namespace kagome::injector { return pimpl_->injector_.create>(); } + std::shared_ptr + KagomeNodeInjector::injectBlockTree() { + return pimpl_->injector_.create>(); + } + + std::shared_ptr KagomeNodeInjector::injectExecutor() { + return pimpl_->injector_.create>(); + + } + + std::shared_ptr KagomeNodeInjector::injectStorage() { + return pimpl_->injector_.create>(); + + } + } // namespace kagome::injector diff --git a/core/injector/application_injector.hpp b/core/injector/application_injector.hpp index a52b90875f..176550fe1b 100644 --- a/core/injector/application_injector.hpp +++ b/core/injector/application_injector.hpp @@ -11,6 +11,7 @@ #include #include "clock/clock.hpp" +#include "storage/buffer_map_types.hpp" namespace soralog { class LoggingSystem; @@ -40,6 +41,11 @@ namespace kagome { class SyncProtocolObserver; } // namespace network + namespace runtime { + class Executor; + } + + namespace api { class ApiService; } @@ -54,6 +60,7 @@ namespace kagome { namespace blockchain { class BlockStorage; + class BlockTree; } namespace storage::trie { @@ -93,6 +100,9 @@ namespace kagome::injector { std::shared_ptr injectTrieStorage(); std::shared_ptr injectMetricsWatcher(); std::shared_ptr injectTelemetryService(); + std::shared_ptr injectBlockTree(); + std::shared_ptr injectExecutor(); + std::shared_ptr injectStorage(); std::shared_ptr injectPrintChainInfoMode(); diff --git a/core/primitives/CMakeLists.txt b/core/primitives/CMakeLists.txt index ae2c9b9875..9d7a4ba661 100644 --- a/core/primitives/CMakeLists.txt +++ b/core/primitives/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(primitives arithmetic_error.cpp author_api_primitives.hpp block.hpp + block_header.cpp block_header.hpp block_id.hpp common.hpp diff --git a/core/primitives/block_header.cpp b/core/primitives/block_header.cpp new file mode 100644 index 0000000000..ca7e2ccd8c --- /dev/null +++ b/core/primitives/block_header.cpp @@ -0,0 +1,11 @@ +#include "primitives/block_header.hpp" + +namespace kagome::primitives { + + outcome::result calculateBlockHash( + BlockHeader const &header, crypto::Hasher const &hasher) { + OUTCOME_TRY(enc_header, scale::encode(header)); + return hasher.blake2b_256(enc_header); + } + +} \ No newline at end of file diff --git a/core/primitives/block_header.hpp b/core/primitives/block_header.hpp index 92af933398..2e3f5d6036 100644 --- a/core/primitives/block_header.hpp +++ b/core/primitives/block_header.hpp @@ -9,7 +9,10 @@ #include #include +#include + #include "common/blob.hpp" +#include "crypto/hasher.hpp" #include "primitives/common.hpp" #include "primitives/compact_integer.hpp" #include "primitives/digest.hpp" @@ -75,6 +78,10 @@ namespace kagome::primitives { bh.number = number_compact.convert_to(); return s; } + + outcome::result calculateBlockHash( + BlockHeader const &header, crypto::Hasher const &hasher); + } // namespace kagome::primitives #endif // KAGOME_PRIMITIVES_BLOCK_HEADER_HPP diff --git a/core/runtime/binaryen/core_api_factory_impl.cpp b/core/runtime/binaryen/core_api_factory_impl.cpp index b3cce84b03..617a7c597f 100644 --- a/core/runtime/binaryen/core_api_factory_impl.cpp +++ b/core/runtime/binaryen/core_api_factory_impl.cpp @@ -37,7 +37,7 @@ namespace kagome::runtime::binaryen { } private: - std::shared_ptr instance_; + std::shared_ptr instance_; std::shared_ptr env_factory_; const std::vector &code_; }; diff --git a/core/runtime/binaryen/module/module_instance_impl.cpp b/core/runtime/binaryen/module/module_instance_impl.cpp index cab08a2270..4c89136f9a 100644 --- a/core/runtime/binaryen/module/module_instance_impl.cpp +++ b/core/runtime/binaryen/module/module_instance_impl.cpp @@ -161,11 +161,5 @@ namespace kagome::runtime::binaryen { segment.data.size())); } } - void ModuleInstanceImpl::borrow( - BorrowedInstance::PoolReleaseFunction release) { - // Releaser for OCWs - doesn't need a valid pointer - static thread_local ModuleInstance::BorrowedInstance pool_release_token{ - nullptr, release}; - } } // namespace kagome::runtime::binaryen diff --git a/core/runtime/binaryen/module/module_instance_impl.hpp b/core/runtime/binaryen/module/module_instance_impl.hpp index db0b45b059..24259aa76b 100644 --- a/core/runtime/binaryen/module/module_instance_impl.hpp +++ b/core/runtime/binaryen/module/module_instance_impl.hpp @@ -41,7 +41,6 @@ namespace kagome::runtime::binaryen { InstanceEnvironment const &getEnvironment() const override; outcome::result resetEnvironment() override; - void borrow(BorrowedInstance::PoolReleaseFunction release) override; void forDataSegment(DataSegmentProcessor const &callback) const override; diff --git a/core/runtime/common/runtime_instances_pool.cpp b/core/runtime/common/runtime_instances_pool.cpp index c4bed1ceb5..d7e2db4795 100644 --- a/core/runtime/common/runtime_instances_pool.cpp +++ b/core/runtime/common/runtime_instances_pool.cpp @@ -1,5 +1,3 @@ - - /** * Copyright Soramitsu Co., Ltd. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 @@ -15,14 +13,53 @@ #include "runtime/runtime_upgrade_tracker.hpp" namespace kagome::runtime { - using kagome::primitives::ThreadNumber; - using soralog::util::getThreadNumber; + /** + * @brief Wrapper type over sptr. Allows to return instance + * back to the ModuleInstancePool upon destruction of + * BorrowedInstance. + */ + class BorrowedInstance : public ModuleInstance { + public: + BorrowedInstance(std::weak_ptr pool, + const RuntimeInstancesPool::RootHash &state, + std::shared_ptr instance) + : pool_{std::move(pool)}, + state_{state}, + instance_{std::move(instance)} {} + ~BorrowedInstance() { + if (auto pool = pool_.lock()) { + pool->release(state_, std::move(instance_)); + } + } + + outcome::result callExportFunction( + std::string_view name, common::BufferView encoded_args) const override { + return instance_->callExportFunction(name, encoded_args); + } + outcome::result> getGlobal( + std::string_view name) const override { + return instance_->getGlobal(name); + } + void forDataSegment(DataSegmentProcessor const &callback) const override { + return instance_->forDataSegment(callback); + } + InstanceEnvironment const &getEnvironment() const override { + return instance_->getEnvironment(); + } + outcome::result resetEnvironment() override { + return instance_->resetEnvironment(); + } + + private: + std::weak_ptr pool_; + RuntimeInstancesPool::RootHash state_; + std::shared_ptr instance_; + }; outcome::result> RuntimeInstancesPool::tryAcquire( const RuntimeInstancesPool::RootHash &state) { std::scoped_lock guard{mt_}; - auto tid = getThreadNumber(); auto &pool = pools_[state]; if (not pool.empty()) { @@ -42,16 +79,12 @@ namespace kagome::runtime { } void RuntimeInstancesPool::release( - const RuntimeInstancesPool::RootHash &state) { + const RuntimeInstancesPool::RootHash &state, + std::shared_ptr &&instance) { std::lock_guard guard{mt_}; - auto tid = getThreadNumber(); auto &pool = pools_[state]; - // if used instance found, release - auto node = pool.extract(tid); - BOOST_ASSERT(node); - node.key() = POOL_FREE_INSTANCE_ID; - pool.insert(std::move(node)); + pool.emplace(std::move(instance)); } std::optional> RuntimeInstancesPool::getModule( const RuntimeInstancesPool::RootHash &state) { diff --git a/core/runtime/common/runtime_instances_pool.hpp b/core/runtime/common/runtime_instances_pool.hpp index 3e19f65a8b..eb1c80f048 100644 --- a/core/runtime/common/runtime_instances_pool.hpp +++ b/core/runtime/common/runtime_instances_pool.hpp @@ -1,148 +1,141 @@ /** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ +* Copyright Soramitsu Co., Ltd. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ #ifndef KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP #define KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP #include "runtime/module_repository.hpp" -#include -#include - -#include "log/logger.hpp" -#include "runtime/instance_environment.hpp" +#include namespace kagome::runtime { - using kagome::primitives::ThreadNumber; - - /** - * LRU cache designed for small amounts of data (as its get() is O(N)) - */ - template - struct SmallLruCache final { - public: - static_assert(std::is_unsigned_v); - - struct CacheEntry { - Key key; - Value value; - PriorityType latest_use_tick_; - - bool operator<(const CacheEntry &rhs) const { - return latest_use_tick_ < rhs.latest_use_tick_; - } - }; - - SmallLruCache(size_t max_size) : kMaxSize{max_size} { - BOOST_ASSERT(kMaxSize > 0); - cache_.reserve(kMaxSize); - } - - std::optional> get(const Key &key) { - ticks_++; - if (ticks_ == 0) { - handleTicksOverflow(); - } - for (auto &entry : cache_) { - if (entry.key == key) { - entry.latest_use_tick_ = ticks_; - return entry.value; - } - } - return std::nullopt; - } - - template - void put(const Key &key, ValueArg &&value) { - static_assert(std::is_convertible_v< - ValueArg, - Value> || std::is_constructible_v); - ticks_++; - if (cache_.size() >= kMaxSize) { - auto min = std::min_element(cache_.begin(), cache_.end()); - cache_.erase(min); - } - cache_.push_back(CacheEntry{key, std::forward(value), ticks_}); - } - - private: - void handleTicksOverflow() { - // 'compress' timestamps of entries in the cache (works because we care - // only about their order, not actual timestamps) - std::sort(cache_.begin(), cache_.end()); - for (auto &entry : cache_) { - entry.latest_use_tick_ = ticks_; - ticks_++; - } - } - - const size_t kMaxSize; - // an abstract representation of time to implement 'recency' without - // depending on real time. Incremented on each cache access - PriorityType ticks_{}; - std::vector cache_; - }; - - /** - * @brief Pool of runtime instances - per state. Incapsulates modules cache. - * - */ - class RuntimeInstancesPool final - : public std::enable_shared_from_this { - using RootHash = storage::trie::RootHash; - using ModuleInstancePool = - std::multimap>; - - public: - using ModuleCache = - SmallLruCache>; - - /** - * @brief @brief Attempt to acquire a ModuleInstance for the provided state. - * If none available, instantiate a new one. If already acquired by this - * thread, return the ptr to the acquired instance. - * - * @param state - the merkle trie root of the state containing the code of - * the runtime module we are acquiring an instance of. - * @return pointer to the acquired ModuleInstance if success. Error - * otherwise. - */ - outcome::result> tryAcquire( - const RootHash &state); - /** - * @brief Releases the module instance (returns it to the pool) - * - * @param state - the merkle trie root of the state containing the runtime - * module code we are releasing an instance of. - */ - void release(const RootHash &state); - - /** - * @brief Get the module for state from internal cache - * - * @param state - the state containing the module's code. - * @return Module if any, nullopt otherwise - */ - std::optional> getModule(const RootHash &state); - - /** - * @brief Puts new module into internal cache - * - * @param state - runtime block, by its root hash - * @param module - new module pointer - */ - void putModule(const RootHash &state, std::shared_ptr module); - - private: - std::mutex mt_; - static constexpr size_t MODULES_CACHE_SIZE = 2; - static constexpr size_t POOL_FREE_INSTANCE_ID = 0; - ModuleCache modules_{MODULES_CACHE_SIZE}; - std::map pools_; - }; + /** + * LRU cache designed for small amounts of data (as its get() is O(N)) + */ + template + struct SmallLruCache final { + public: + static_assert(std::is_unsigned_v); + + struct CacheEntry { + Key key; + Value value; + PriorityType latest_use_tick_; + + bool operator<(const CacheEntry &rhs) const { + return latest_use_tick_ < rhs.latest_use_tick_; + } + }; + + SmallLruCache(size_t max_size) : kMaxSize{max_size} { + BOOST_ASSERT(kMaxSize > 0); + cache_.reserve(kMaxSize); + } + + std::optional> get(const Key &key) { + ticks_++; + if (ticks_ == 0) { + handleTicksOverflow(); + } + for (auto &entry : cache_) { + if (entry.key == key) { + entry.latest_use_tick_ = ticks_; + return entry.value; + } + } + return std::nullopt; + } + + template + void put(const Key &key, ValueArg &&value) { + static_assert(std::is_convertible_v< + ValueArg, + Value> || std::is_constructible_v); + ticks_++; + if (cache_.size() >= kMaxSize) { + auto min = std::min_element(cache_.begin(), cache_.end()); + cache_.erase(min); + } + cache_.push_back(CacheEntry{key, std::forward(value), ticks_}); + } + + private: + void handleTicksOverflow() { + // 'compress' timestamps of entries in the cache (works because we care + // only about their order, not actual timestamps) + std::sort(cache_.begin(), cache_.end()); + for (auto &entry : cache_) { + entry.latest_use_tick_ = ticks_; + ticks_++; + } + } + + const size_t kMaxSize; + // an abstract representation of time to implement 'recency' without + // depending on real time. Incremented on each cache access + PriorityType ticks_{}; + std::vector cache_; + }; + + /** + * @brief Pool of runtime instances - per state. Incapsulates modules cache. + * + */ + class RuntimeInstancesPool final + : public std::enable_shared_from_this { + using ModuleInstancePool = std::stack>; + + public: + using RootHash = storage::trie::RootHash; + using ModuleCache = + SmallLruCache>; + + /** + * @brief Instantiate new or reuse existing ModuleInstance for the provided + * state. + * + * @param state - the merkle trie root of the state containing the code of + * the runtime module we are acquiring an instance of. + * @return pointer to the acquired ModuleInstance if success. Error + * otherwise. + */ + outcome::result> tryAcquire( + const RootHash &state); + /** + * @brief Releases the module instance (returns it to the pool) + * + * @param state - the merkle trie root of the state containing the runtime + * module code we are releasing an instance of. + * @param instance - instance to be released. + */ + void release(const RootHash &state, + std::shared_ptr &&instance); + + /** + * @brief Get the module for state from internal cache + * + * @param state - the state containing the module's code. + * @return Module if any, nullopt otherwise + */ + std::optional> getModule(const RootHash &state); + + /** + * @brief Puts new module into internal cache + * + * @param state - runtime block, by its root hash + * @param module - new module pointer + */ + void putModule(const RootHash &state, std::shared_ptr module); + + private: + std::mutex mt_; + static constexpr size_t MODULES_CACHE_SIZE = 2; + ModuleCache modules_{MODULES_CACHE_SIZE}; + std::map pools_; + }; } // namespace kagome::runtime -#endif // KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP +#endif // KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP \ No newline at end of file diff --git a/core/runtime/module_instance.hpp b/core/runtime/module_instance.hpp index 485bd9e956..0f96781ab7 100644 --- a/core/runtime/module_instance.hpp +++ b/core/runtime/module_instance.hpp @@ -31,36 +31,6 @@ namespace kagome::runtime { public: virtual ~ModuleInstance() = default; - /** - * @brief Wrapper type over sptr. Allows to return instance - * back to the ModuleInstancePool upon destruction of - * BorrowedInstance. - */ - class BorrowedInstance { - public: - using PoolReleaseFunction = std::function; - - BorrowedInstance(std::shared_ptr instance, - PoolReleaseFunction cache_release = {}) - : instance_{std::move(instance)}, - cache_release_{std::move(cache_release)} {} - ~BorrowedInstance() { - if (cache_release_) { - cache_release_(); - } - } - bool operator==(std::nullptr_t) { - return instance_ == nullptr; - } - ModuleInstance *operator->() { - return instance_.get(); - } - - private: - std::shared_ptr instance_; - PoolReleaseFunction cache_release_; - }; - /** * Call the instance's function * @param name - name of the function @@ -82,14 +52,6 @@ namespace kagome::runtime { virtual InstanceEnvironment const &getEnvironment() const = 0; virtual outcome::result resetEnvironment() = 0; - - /** - * @brief Make thread borrow a wrapped pointer to this instance with custom - * deleter 'release' - * - * @param release - a deleter, that should be called upon thread destruction - */ - virtual void borrow(BorrowedInstance::PoolReleaseFunction release) = 0; }; } // namespace kagome::runtime diff --git a/core/runtime/wavm/core_api_factory_impl.cpp b/core/runtime/wavm/core_api_factory_impl.cpp index 0a48762038..b4aae0d842 100644 --- a/core/runtime/wavm/core_api_factory_impl.cpp +++ b/core/runtime/wavm/core_api_factory_impl.cpp @@ -43,7 +43,7 @@ namespace kagome::runtime::wavm { BOOST_ASSERT(last_compiled_module_); } - outcome::result> getInstanceAt( + outcome::result> getInstanceAt( std::shared_ptr, const primitives::BlockInfo &, const primitives::BlockHeader &) override { @@ -61,7 +61,7 @@ namespace kagome::runtime::wavm { } private: - std::shared_ptr instance_; + std::shared_ptr instance_; std::shared_ptr instance_env_factory_; std::shared_ptr compartment_; std::shared_ptr module_params_; diff --git a/core/runtime/wavm/instance_environment_factory.hpp b/core/runtime/wavm/instance_environment_factory.hpp index a0b1033fb0..628859c82f 100644 --- a/core/runtime/wavm/instance_environment_factory.hpp +++ b/core/runtime/wavm/instance_environment_factory.hpp @@ -34,8 +34,6 @@ namespace kagome::runtime { } namespace kagome::runtime::wavm { - - class ModuleInstance; class IntrinsicModule; class IntrinsicModuleInstance; class IntrinsicResolver; diff --git a/core/runtime/wavm/intrinsics/intrinsic_functions.cpp b/core/runtime/wavm/intrinsics/intrinsic_functions.cpp index 32dfdc24d3..e07a891696 100644 --- a/core/runtime/wavm/intrinsics/intrinsic_functions.cpp +++ b/core/runtime/wavm/intrinsics/intrinsic_functions.cpp @@ -12,24 +12,26 @@ namespace kagome::runtime::wavm { log::Logger logger; - static thread_local std::stack< - std::shared_ptr> + static thread_local std::stack> global_instances; void pushBorrowedRuntimeInstance( - std::shared_ptr - borrowed_runtime_instance) { + std::shared_ptr borrowed_runtime_instance) { global_instances.emplace(std::move(borrowed_runtime_instance)); } - std::shared_ptr - peekBorrowedRuntimeInstance() { + void popBorrowedRuntimeInstance() { + BOOST_ASSERT(!global_instances.empty()); + global_instances.pop(); + } + + std::shared_ptr peekBorrowedRuntimeInstance() { BOOST_ASSERT(!global_instances.empty()); return global_instances.top(); } std::shared_ptr peekHostApi() { - return (*peekBorrowedRuntimeInstance())->getEnvironment().host_api; + return peekBorrowedRuntimeInstance()->getEnvironment().host_api; } #undef WAVM_DEFINE_INTRINSIC_FUNCTION diff --git a/core/runtime/wavm/intrinsics/intrinsic_functions.hpp b/core/runtime/wavm/intrinsics/intrinsic_functions.hpp index e9f58d8a70..8cbfd05e96 100644 --- a/core/runtime/wavm/intrinsics/intrinsic_functions.hpp +++ b/core/runtime/wavm/intrinsics/intrinsic_functions.hpp @@ -26,10 +26,9 @@ namespace kagome::runtime::wavm { std::shared_ptr peekHostApi(); void pushBorrowedRuntimeInstance( - std::shared_ptr - borrowed_runtime_instance); - std::shared_ptr - peekBorrowedRuntimeInstance(); + std::shared_ptr borrowed_runtime_instance); + void popBorrowedRuntimeInstance(); + std::shared_ptr peekBorrowedRuntimeInstance(); extern log::Logger logger; diff --git a/core/runtime/wavm/module.cpp b/core/runtime/wavm/module.cpp index 78ad40c64e..a8aab8b758 100644 --- a/core/runtime/wavm/module.cpp +++ b/core/runtime/wavm/module.cpp @@ -72,8 +72,8 @@ namespace kagome::runtime::wavm { BOOST_ASSERT(module_); } - outcome::result> - ModuleImpl::instantiate() const { + outcome::result> ModuleImpl::instantiate() + const { const auto &ir_module = WAVM::Runtime::getModuleIR(module_); bool imports_memory = std::find_if(ir_module.imports.cbegin(), @@ -102,7 +102,7 @@ namespace kagome::runtime::wavm { auto env = env_factory_->make( memory_origin, internal_instance, new_intrinsic_module_instance); - auto instance = std::make_shared( + auto instance = std::make_shared( std::move(env), internal_instance, module_, compartment_); return instance; diff --git a/core/runtime/wavm/module.hpp b/core/runtime/wavm/module.hpp index 4a8b9e3eb3..71eccb55b4 100644 --- a/core/runtime/wavm/module.hpp +++ b/core/runtime/wavm/module.hpp @@ -36,8 +36,8 @@ namespace kagome::runtime::wavm { std::shared_ptr env_factory, gsl::span code); - outcome::result> - instantiate() const override; + outcome::result> instantiate() + const override; private: ModuleImpl(std::shared_ptr compartment, diff --git a/core/runtime/wavm/module_instance.cpp b/core/runtime/wavm/module_instance.cpp index 3125102626..ef91035de9 100644 --- a/core/runtime/wavm/module_instance.cpp +++ b/core/runtime/wavm/module_instance.cpp @@ -62,8 +62,10 @@ static WAVM::Uptr getIndexValue(const WAVM::IR::Value &value, }; } -OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wavm, ModuleInstance::Error, e) { - using E = kagome::runtime::wavm::ModuleInstance::Error; +OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wavm, + ModuleInstanceImpl::Error, + e) { + using E = kagome::runtime::wavm::ModuleInstanceImpl::Error; switch (e) { case E::WRONG_ARG_COUNT: return "The provided function argument count should equal to 2"; @@ -80,7 +82,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::runtime::wavm, ModuleInstance::Error, e) { namespace kagome::runtime::wavm { - ModuleInstance::ModuleInstance( + ModuleInstanceImpl::ModuleInstanceImpl( InstanceEnvironment &&env, WAVM::Runtime::GCPointer instance, WAVM::Runtime::ModuleRef module, @@ -95,7 +97,7 @@ namespace kagome::runtime::wavm { BOOST_ASSERT(module_ != nullptr); } - outcome::result ModuleInstance::callExportFunction( + outcome::result ModuleInstanceImpl::callExportFunction( std::string_view name, common::BufferView encoded_args) const { auto memory = env_.memory_provider->getCurrentMemory().value(); @@ -132,6 +134,9 @@ namespace kagome::runtime::wavm { // Allocate an array to receive the invocation results. BOOST_ASSERT(invokeSig.results().size() == 1); std::array untaggedInvokeResults; + pushBorrowedRuntimeInstance( + std::const_pointer_cast(shared_from_this())); + const auto pop = gsl::finally(&popBorrowedRuntimeInstance); try { WAVM::Runtime::unwindSignalsAsExceptions( [&context, @@ -160,7 +165,7 @@ namespace kagome::runtime::wavm { return res; } - outcome::result> ModuleInstance::getGlobal( + outcome::result> ModuleInstanceImpl::getGlobal( std::string_view name) const { auto global = WAVM::Runtime::asGlobalNullable( WAVM::Runtime::getInstanceExport(instance_, name.data())); @@ -185,7 +190,7 @@ namespace kagome::runtime::wavm { } } - void ModuleInstance::forDataSegment( + void ModuleInstanceImpl::forDataSegment( DataSegmentProcessor const &callback) const { using WAVM::Uptr; using WAVM::IR::DataSegment; @@ -207,19 +212,12 @@ namespace kagome::runtime::wavm { } } - InstanceEnvironment const &ModuleInstance::getEnvironment() const { + InstanceEnvironment const &ModuleInstanceImpl::getEnvironment() const { return env_; } - outcome::result ModuleInstance::resetEnvironment() { + outcome::result ModuleInstanceImpl::resetEnvironment() { env_.host_api->reset(); return outcome::success(); } - - void ModuleInstance::borrow(BorrowedInstance::PoolReleaseFunction release) { - auto borrowed = std::make_shared( - shared_from_this(), release); - pushBorrowedRuntimeInstance(std::move(borrowed)); - } - } // namespace kagome::runtime::wavm diff --git a/core/runtime/wavm/module_instance.hpp b/core/runtime/wavm/module_instance.hpp index 17a16e6760..fe8449d5a6 100644 --- a/core/runtime/wavm/module_instance.hpp +++ b/core/runtime/wavm/module_instance.hpp @@ -26,8 +26,9 @@ namespace kagome::runtime::wavm { class CompartmentWrapper; - class ModuleInstance : public runtime::ModuleInstance, - public std::enable_shared_from_this { + class ModuleInstanceImpl + : public ModuleInstance, + public std::enable_shared_from_this { public: enum class Error { FUNC_NOT_FOUND = 1, @@ -35,10 +36,11 @@ namespace kagome::runtime::wavm { EXECUTION_ERROR, WRONG_RETURN_TYPE }; - ModuleInstance(InstanceEnvironment &&env, - WAVM::Runtime::GCPointer instance, - WAVM::Runtime::ModuleRef module, - std::shared_ptr compartment); + ModuleInstanceImpl( + InstanceEnvironment &&env, + WAVM::Runtime::GCPointer instance, + WAVM::Runtime::ModuleRef module, + std::shared_ptr compartment); outcome::result callExportFunction( std::string_view name, common::BufferView encoded_args) const override; @@ -50,7 +52,6 @@ namespace kagome::runtime::wavm { InstanceEnvironment const &getEnvironment() const override; outcome::result resetEnvironment() override; - void borrow(BorrowedInstance::PoolReleaseFunction release) override; private: InstanceEnvironment env_; @@ -62,6 +63,6 @@ namespace kagome::runtime::wavm { } // namespace kagome::runtime::wavm -OUTCOME_HPP_DECLARE_ERROR(kagome::runtime::wavm, ModuleInstance::Error) +OUTCOME_HPP_DECLARE_ERROR(kagome::runtime::wavm, ModuleInstanceImpl::Error) #endif // KAGOME_CORE_RUNTIME_WAVM_IMPL_MODULE_INSTANCE_HPP diff --git a/core/runtime/wavm/wavm_internal_memory_provider.hpp b/core/runtime/wavm/wavm_internal_memory_provider.hpp index 4846d94cda..2e9ca8a536 100644 --- a/core/runtime/wavm/wavm_internal_memory_provider.hpp +++ b/core/runtime/wavm/wavm_internal_memory_provider.hpp @@ -13,8 +13,6 @@ namespace WAVM::Runtime { } namespace kagome::runtime::wavm { - - class ModuleInstance; class MemoryImpl; class WavmInternalMemoryProvider final : public MemoryProvider { diff --git a/core/utils/CMakeLists.txt b/core/utils/CMakeLists.txt index b8b922a1ac..6a46bcd66d 100644 --- a/core/utils/CMakeLists.txt +++ b/core/utils/CMakeLists.txt @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # -add_executable(storage_explorer storage_explorer.cpp) +add_executable(storage_explorer storage_explorer.cpp ${BACKWARD_ENABLE}) target_link_libraries(storage_explorer application_injector) +add_backward(storage_explorer) add_executable(kagome-db-editor kagome_db_editor.cpp diff --git a/core/utils/storage_explorer.cpp b/core/utils/storage_explorer.cpp index f8114de42d..08c073447f 100644 --- a/core/utils/storage_explorer.cpp +++ b/core/utils/storage_explorer.cpp @@ -9,15 +9,24 @@ #include "application/impl/app_configuration_impl.hpp" #include "blockchain/block_storage.hpp" #include "blockchain/block_tree.hpp" +#include "blockchain/impl/block_header_repository_impl.hpp" +#include "blockchain/impl/block_tree_impl.hpp" +#include "consensus/authority/impl/authority_manager_impl.hpp" +#include "crypto/hasher/hasher_impl.hpp" #include "injector/application_injector.hpp" #include "log/configurator.hpp" +#include "runtime/runtime_api/impl/grandpa_api.hpp" #include "storage/trie/trie_storage.hpp" +using kagome::authority::AuthorityManagerImpl; using kagome::blockchain::BlockStorage; +using kagome::crypto::Hasher; +using kagome::crypto::HasherImpl; using kagome::primitives::BlockHeader; using kagome::primitives::BlockId; using kagome::primitives::BlockNumber; using kagome::primitives::GrandpaDigest; +using kagome::runtime::GrandpaApi; using kagome::storage::trie::TrieStorage; using ArgumentList = gsl::span; @@ -185,6 +194,18 @@ class InspectBlockCommand : public Command { std::cout << "State root: " << res.value()->state_root.toHex() << "\n"; std::cout << "Extrinsics root: " << res.value()->extrinsics_root.toHex() << "\n"; + + if (auto body_res = block_storage->getBlockBody(opt_id.value()); + body_res) { + if (!body_res.value().has_value()) { + throwError("Block body not found for '{}'", args[1]); + } + std::cout << "# of extrinsics: " << body_res.value()->size() << "\n"; + + } else { + throwError("Internal error: {}}", body_res.error().message()); + } + } else { throwError("Internal error: {}}", res.error().message()); } @@ -272,12 +293,23 @@ class QueryStateCommand : public Command { class SearchChainCommand : public Command { public: - explicit SearchChainCommand(std::shared_ptr block_storage) + explicit SearchChainCommand( + std::shared_ptr block_storage, + std::shared_ptr trie_storage, + std::shared_ptr authority_manager, + std::shared_ptr hasher) : Command{"search-chain", "target [start block/0] [end block/deepest finalized] - search " "the finalized chain for the target entity. Currently, " "'justification' or 'authority-update' are supported "}, - block_storage{block_storage} {} + block_storage{block_storage}, + trie_storage{trie_storage}, + hasher{hasher} { + BOOST_ASSERT(block_storage != nullptr); + BOOST_ASSERT(trie_storage != nullptr); + BOOST_ASSERT(authority_manager != nullptr); + BOOST_ASSERT(hasher != nullptr); + } enum class Target { Justification, AuthorityUpdate, LastBlock }; @@ -328,9 +360,9 @@ class SearchChainCommand : public Command { } auto &end_header = end_header_opt.value(); - for (int64_t current = end_header.number, stop = start_header.number; - current >= stop; - current--) { + for (int64_t current = start_header.number, stop = end_header.number; + current <= stop; + current++) { auto current_header_opt = unwrapResult(fmt::format("Getting header of block #{}", current), block_storage->getBlockHeader(current)); @@ -383,18 +415,36 @@ class SearchChainCommand : public Command { consensus_digest->decode()); if (decoded.consensus_engine_id == kagome::primitives::kGrandpaEngineId) { - reportAuthorityUpdate(out, header.number, decoded.asGrandpaDigest()); + auto info = reportAuthorityUpdate( + out, header.number, decoded.asGrandpaDigest()); + if (info.id_changed) { + auto res = kagome::authority::fetchSetIdFromTrieStorage( + *trie_storage, *hasher, header.state_root); + if (!res) { + std::cerr << "Error fetching authority set id from storage: " + << res.error().message() << "\n"; + continue; + } + std::cout << "Set id fetched from storage: " + << res.value().value_or(-1) << "\n"; + } } } } } - void reportAuthorityUpdate(std::ostream &out, - BlockNumber digest_origin, - GrandpaDigest const &digest) const { + struct AuthorityUpdateInfo { + bool id_changed = false; + }; + + AuthorityUpdateInfo reportAuthorityUpdate(std::ostream &out, + BlockNumber digest_origin, + GrandpaDigest const &digest) const { using namespace kagome::primitives; + bool has_id_change = false; if (auto *scheduled_change = boost::get(&digest); scheduled_change) { + has_id_change = true; out << "ScheduledChange at #" << digest_origin << " for "; if (scheduled_change->subchain_length > 0) { out << "#" << digest_origin + scheduled_change->subchain_length; @@ -405,12 +455,11 @@ class SearchChainCommand : public Command { } else if (auto *forced_change = boost::get(&digest); forced_change) { - out << "ForcedChange at " << digest_origin << " for "; - if (forced_change->subchain_length > 0) { - out << "#" << digest_origin + forced_change->subchain_length; - } else { - out << "the same block"; - } + has_id_change = true; + out << "ForcedChange at " << digest_origin << ", delay starts at #" + << forced_change->delay_start << " for " + << forced_change->subchain_length << " blocks (so activates at #" + << forced_change->delay_start + forced_change->subchain_length << ")"; out << "\n"; } else if (auto *pause = boost::get(&digest); pause) { @@ -425,9 +474,12 @@ class SearchChainCommand : public Command { out << "Disabled at " << digest_origin << " for authority " << disabled->authority_index << "\n"; } + return {.id_changed = has_id_change}; } std::shared_ptr block_storage; + std::shared_ptr trie_storage; + std::shared_ptr hasher; }; int main(int argc, const char **argv) { @@ -473,12 +525,35 @@ int main(int argc, const char **argv) { kagome::injector::KagomeNodeInjector injector{configuration}; auto block_storage = injector.injectBlockStorage(); - auto trie_storage = injector.injectTrieStorage(); + auto app_state_manager = injector.injectAppStateManager(); + auto block_tree = injector.injectBlockTree(); + auto executor = injector.injectExecutor(); + auto persistent_storage = injector.injectStorage(); + auto hasher = std::make_shared(); + + auto header_repo = + std::make_shared( + persistent_storage, hasher); + auto grandpa_api = + std::make_shared(header_repo, executor); + + auto authority_manager = + std::make_shared( + kagome::authority::AuthorityManagerImpl::Config{}, + app_state_manager, + block_tree, + trie_storage, + grandpa_api, + hasher, + persistent_storage, + header_repo); + parser.addCommand(std::make_unique(block_storage)); parser.addCommand(std::make_unique(block_storage)); parser.addCommand(std::make_unique(trie_storage)); - parser.addCommand(std::make_unique(block_storage)); + parser.addCommand(std::make_unique( + block_storage, trie_storage, authority_manager, hasher)); parser.invoke(args.subspan(0, kagome_args_start)); diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index 7640f7a151..61bb196c73 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -10,15 +10,17 @@ #include "consensus/authority/impl/schedule_node.hpp" #include "mock/core/application/app_state_manager_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" +#include "mock/core/blockchain/block_header_repository_mock.hpp" #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/runtime/grandpa_api_mock.hpp" +#include "mock/core/runtime/runtime_environment_factory_mock.hpp" #include "mock/core/storage/persistent_map_mock.hpp" #include "mock/core/storage/trie/trie_batches_mock.hpp" #include "mock/core/storage/trie/trie_storage_mock.hpp" #include "primitives/digest.hpp" -#include "scale/scale.hpp" #include "storage/in_memory/in_memory_storage.hpp" #include "storage/predefined_keys.hpp" +#include "runtime/executor.hpp" #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/outcome/dummy_error.hpp" @@ -80,8 +82,6 @@ class AuthorityManagerTest : public testing::Test { EXPECT_CALL(*app_state_manager, atPrepare(_)); - auto executore = std::make_shared(); - authority_manager = std::make_shared(AuthorityManagerImpl::Config{}, app_state_manager, @@ -89,7 +89,8 @@ class AuthorityManagerTest : public testing::Test { trie_storage, grandpa_api, hasher, - persistent_storage); + persistent_storage, + header_repo); static auto genesis_hash = "genesis"_hash256; ON_CALL(*block_tree, getGenesisBlockHash()) diff --git a/test/core/consensus/grandpa/voting_round/voting_round_test.cpp b/test/core/consensus/grandpa/voting_round/voting_round_test.cpp index 8a95f6d65f..cda63ac636 100644 --- a/test/core/consensus/grandpa/voting_round/voting_round_test.cpp +++ b/test/core/consensus/grandpa/voting_round/voting_round_test.cpp @@ -38,6 +38,7 @@ using kagome::crypto::Ed25519Signature; using kagome::crypto::HasherMock; using kagome::primitives::Authority; using kagome::primitives::AuthorityList; +using kagome::primitives::AuthoritySet; using Propagation = kagome::consensus::grandpa::VotingRound::Propagation; using kagome::visit_in_place; @@ -115,11 +116,11 @@ class VotingRoundTest : public testing::Test, EXPECT_CALL(*grandpa_, tryExecuteNextRound(_)).Times(AnyNumber()); EXPECT_CALL(*grandpa_, updateNextRound(_)).Times(AnyNumber()); - auto authorities = std::make_shared(); + auto authorities = std::make_shared(); authorities->id = 0; - authorities->emplace_back(Authority{{kAlice}, kAliceWeight}); - authorities->emplace_back(Authority{{kBob}, kBobWeight}); - authorities->emplace_back(Authority{{kEve}, kEveWeight}); + authorities->authorities.emplace_back(Authority{{kAlice}, kAliceWeight}); + authorities->authorities.emplace_back(Authority{{kBob}, kBobWeight}); + authorities->authorities.emplace_back(Authority{{kEve}, kEveWeight}); authority_manager_ = std::make_shared(); EXPECT_CALL(*authority_manager_, base()) diff --git a/test/mock/core/runtime/grandpa_api_mock.hpp b/test/mock/core/runtime/grandpa_api_mock.hpp index d69ce28742..82a5730e51 100644 --- a/test/mock/core/runtime/grandpa_api_mock.hpp +++ b/test/mock/core/runtime/grandpa_api_mock.hpp @@ -28,6 +28,11 @@ namespace kagome::runtime { authorities, (const primitives::BlockId &block_id), (override)); + + MOCK_METHOD(outcome::result, + current_set_id, + (const primitives::BlockHash &block), + (override)); }; } // namespace kagome::runtime diff --git a/test/mock/core/runtime/module_instance_mock.hpp b/test/mock/core/runtime/module_instance_mock.hpp index b8e9ee3058..0134dd0388 100644 --- a/test/mock/core/runtime/module_instance_mock.hpp +++ b/test/mock/core/runtime/module_instance_mock.hpp @@ -34,8 +34,6 @@ namespace kagome::runtime { (), (const, override)); - MOCK_METHOD(void, borrow, (std::function), (override)); - MOCK_METHOD(outcome::result, resetEnvironment, (), (override)); }; } // namespace kagome::runtime From 224b413b9764f437c8f725d02df7a808bffce116 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 25 Aug 2022 15:10:17 +0300 Subject: [PATCH 59/74] Fix bad merge of test.yml --- .github/workflows/test.yml | 31 +++++++++++-------- .../consensus/authority/authority_manager.hpp | 2 -- core/consensus/authority/impl/CMakeLists.txt | 1 + .../authority/impl/authority_manager_impl.hpp | 4 +-- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b7dc4a21d..c1f418c77f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,17 +26,17 @@ jobs: MacOS: runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: install - run: ./housekeeping/macos/dependency.sh - - name: build - env: - DEVELOPER_DIR: /Applications/Xcode_11.7.app/Contents/Developer - run: ./housekeeping/make_build.sh -DCOVERAGE=OFF -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx17.cmake + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ${{ env.CACHE_PATHS }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }} + - name: install + run: ./housekeeping/macos/dependency.sh + - name: build + env: + DEVELOPER_DIR: /Applications/Xcode_11.7.app/Contents/Developer + run: ./housekeeping/make_build.sh -DCOVERAGE=OFF -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx17.cmake Linux: strategy: @@ -52,10 +52,10 @@ jobs: - name: "Linux: clang-11 UBSAN" run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-11_cxx17.cmake -DUBSAN=ON - name: "Linux: clang-11 External Project" - run: ./housekeeping/make_external_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/clang-11-cpp17-toolchain.cmake + run: ./housekeeping/make_external_build.sh -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-11_cxx17.cmake - name: "Linux: clang-14 UBSAN" run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-14_cxx17.cmake -DUBSAN=ON - + name: "${{ matrix.options.name }}" runs-on: ubuntu-latest container: soramitsu/kagome-dev:4-minideb @@ -65,6 +65,8 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} + - name: Install mold + run: ./housekeeping/ci_install_mold.sh --make-default - name: "${{ matrix.options.name }}" run: "${{ matrix.options.run }}" @@ -98,6 +100,8 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} + - name: Install mold + run: ./housekeeping/ci_install_mold.sh --make-default - name: makeBuild env: BUILD_FINAL_TARGET: ctest_coverage @@ -158,6 +162,7 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} + - run: git fetch --prune --unshallow - name: build env: BUILD_TYPE: "${{ matrix.options.build-type }}" diff --git a/core/consensus/authority/authority_manager.hpp b/core/consensus/authority/authority_manager.hpp index 2cec07f3a5..02b613a279 100644 --- a/core/consensus/authority/authority_manager.hpp +++ b/core/consensus/authority/authority_manager.hpp @@ -33,8 +33,6 @@ namespace kagome::authority { public: virtual ~AuthorityManager() = default; - virtual outcome::result initializeAt(const primitives::BlockInfo& root_block) = 0; - /** * Recalculate the authority change graph starting from genesis and up to * the last finalized block. The result shall be stored in the provided diff --git a/core/consensus/authority/impl/CMakeLists.txt b/core/consensus/authority/impl/CMakeLists.txt index 915a1418e3..270858b4b6 100644 --- a/core/consensus/authority/impl/CMakeLists.txt +++ b/core/consensus/authority/impl/CMakeLists.txt @@ -12,4 +12,5 @@ target_link_libraries(authority_manager authority_manager_error authority_update_observer_error block_tree_error + primitives ) diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 07e6861b4d..c0d244222e 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -70,8 +70,6 @@ namespace kagome::authority { bool prepare(); - outcome::result initializeAt(const primitives::BlockInfo &root_block); - primitives::BlockInfo base() const override; std::optional> authorities( @@ -109,6 +107,8 @@ namespace kagome::authority { void prune(const primitives::BlockInfo &block) override; private: + outcome::result initializeAt(const primitives::BlockInfo &root_block); + /** * @brief Find schedule_node according to the block * @param block for which to find the schedule node From b30f5065792cc1b3cc98b22807820cb0bf8428c7 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 25 Aug 2022 16:46:23 +0300 Subject: [PATCH 60/74] Fix bad merge of test.yml --- .github/workflows/test.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1f418c77f..940ea55986 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,17 +26,17 @@ jobs: MacOS: runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: install - run: ./housekeeping/macos/dependency.sh - - name: build - env: - DEVELOPER_DIR: /Applications/Xcode_11.7.app/Contents/Developer - run: ./housekeeping/make_build.sh -DCOVERAGE=OFF -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx17.cmake + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ${{ env.CACHE_PATHS }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }} + - name: install + run: ./housekeeping/macos/dependency.sh + - name: build + env: + DEVELOPER_DIR: /Applications/Xcode_11.7.app/Contents/Developer + run: ./housekeeping/make_build.sh -DCOVERAGE=OFF -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx17.cmake Linux: strategy: @@ -55,7 +55,7 @@ jobs: run: ./housekeeping/make_external_build.sh -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-11_cxx17.cmake - name: "Linux: clang-14 UBSAN" run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-14_cxx17.cmake -DUBSAN=ON - + name: "${{ matrix.options.name }}" runs-on: ubuntu-latest container: soramitsu/kagome-dev:4-minideb @@ -162,7 +162,6 @@ jobs: with: path: ${{ env.CACHE_PATHS }} key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - run: git fetch --prune --unshallow - name: build env: BUILD_TYPE: "${{ matrix.options.build-type }}" From 8a78055493f15448ac577c8a8937119977b56fc2 Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 26 Aug 2022 11:08:48 +0300 Subject: [PATCH 61/74] Self review --- cmake/print.cmake | 2 +- .../impl/vote_crypto_provider_impl.cpp | 11 +++++-- core/injector/application_injector.cpp | 30 ++++++++----------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/cmake/print.cmake b/cmake/print.cmake index e03e2642b7..7291735b1b 100644 --- a/cmake/print.cmake +++ b/cmake/print.cmake @@ -3,6 +3,6 @@ function(print) endfunction() function(fatal_error) - message(FATAL_ERROOR "[${CMAKE_PROJECT_NAME}] ${ARGV}") + message(FATAL_ERROR "[${CMAKE_PROJECT_NAME}] ${ARGV}") endfunction() diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index 47da1c9a8b..6436c7b89e 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -38,20 +38,25 @@ namespace kagome::consensus::grandpa { auto verifying_result = ed_provider_->verify(vote.signature, payload, vote.id); bool result = verifying_result.has_value() and verifying_result.value(); +#ifndef NDEBUG // proves really useful for debugging voter set and round number + // calculation errors if (!result) { auto logger = log::createLogger("VoteCryptoProvider", "authority"); for (auto id = voter_set_->id() - 50; id < voter_set_->id() + 50; id++) { - auto payload = - scale::encode(vote.message, number, id).value(); + auto payload = scale::encode(vote.message, number, id).value(); auto verifying_result = ed_provider_->verify(vote.signature, payload, vote.id); if (verifying_result.has_value() and verifying_result.value()) { - SL_DEBUG(logger, "Correct set id is {}, actual is {}", id, voter_set_->id()); + SL_DEBUG(logger, + "Correct set id is {}, actual is {}", + id, + voter_set_->id()); return false; } } SL_DEBUG(logger, "Failed to find correct set id"); } +#endif return result; } diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 477859b8d1..17ae60c8fa 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -10,27 +10,16 @@ #define BOOST_DI_CFG_CTOR_LIMIT_SIZE \ 16 // TODO(Harrm): check how it influences on compilation time +#include +#include #include #include #include -#include -#include #include #include #undef U64 // comes from OpenSSL and messes with WAVM -#include "runtime/wavm/compartment_wrapper.hpp" -#include "runtime/wavm/core_api_factory_impl.hpp" -#include "runtime/wavm/instance_environment_factory.hpp" -#include "runtime/wavm/intrinsics/intrinsic_functions.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module.hpp" -#include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" -#include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" -#include "runtime/wavm/module.hpp" -#include "runtime/wavm/module_cache.hpp" -#include "runtime/wavm/module_factory_impl.hpp" - #include "api/service/author/author_jrpc_processor.hpp" #include "api/service/author/impl/author_api_impl.hpp" #include "api/service/chain/chain_jrpc_processor.hpp" @@ -140,6 +129,16 @@ #include "runtime/runtime_api/impl/session_keys_api.hpp" #include "runtime/runtime_api/impl/tagged_transaction_queue.hpp" #include "runtime/runtime_api/impl/transaction_payment_api.hpp" +#include "runtime/wavm/compartment_wrapper.hpp" +#include "runtime/wavm/core_api_factory_impl.hpp" +#include "runtime/wavm/instance_environment_factory.hpp" +#include "runtime/wavm/intrinsics/intrinsic_functions.hpp" +#include "runtime/wavm/intrinsics/intrinsic_module.hpp" +#include "runtime/wavm/intrinsics/intrinsic_module_instance.hpp" +#include "runtime/wavm/intrinsics/intrinsic_resolver_impl.hpp" +#include "runtime/wavm/module.hpp" +#include "runtime/wavm/module_cache.hpp" +#include "runtime/wavm/module_factory_impl.hpp" #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp" #include "storage/predefined_keys.hpp" #include "storage/rocksdb/rocksdb.hpp" @@ -1616,19 +1615,16 @@ namespace kagome::injector { return pimpl_->injector_.create>(); } - std::shared_ptr - KagomeNodeInjector::injectBlockTree() { + std::shared_ptr KagomeNodeInjector::injectBlockTree() { return pimpl_->injector_.create>(); } std::shared_ptr KagomeNodeInjector::injectExecutor() { return pimpl_->injector_.create>(); - } std::shared_ptr KagomeNodeInjector::injectStorage() { return pimpl_->injector_.create>(); - } } // namespace kagome::injector From 6e840f40272164676a2ad930b5f65361bf898b81 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 8 Sep 2022 12:35:08 +0300 Subject: [PATCH 62/74] Improve voting round error logging --- .../babe/impl/block_appender_impl.cpp | 2 +- .../babe/impl/block_executor_impl.cpp | 2 +- core/consensus/grandpa/CMakeLists.txt | 2 +- core/consensus/grandpa/impl/grandpa_impl.cpp | 2 +- .../grandpa/impl/voting_round_impl.cpp | 12 ++++++---- .../grandpa/vote_graph/CMakeLists.txt | 1 + .../grandpa/vote_graph/vote_graph_impl.cpp | 6 +++-- .../grandpa/vote_graph/vote_graph_impl.hpp | 1 + core/consensus/grandpa/voter_set.cpp | 22 ++++++++---------- core/consensus/grandpa/voter_set.hpp | 10 ++++---- .../grandpa/{impl => }/voting_round_error.cpp | 2 +- .../grandpa/{impl => }/voting_round_error.hpp | 2 +- housekeeping/indocker.sh | 5 ++-- test/core/consensus/babe/babe_test.cpp | 7 +++++- .../core/consensus/grandpa/voter_set_test.cpp | 23 ++++++++----------- .../voting_round/voting_round_test.cpp | 2 +- 16 files changed, 53 insertions(+), 48 deletions(-) rename core/consensus/grandpa/{impl => }/voting_round_error.cpp (97%) rename core/consensus/grandpa/{impl => }/voting_round_error.hpp (96%) diff --git a/core/consensus/babe/impl/block_appender_impl.cpp b/core/consensus/babe/impl/block_appender_impl.cpp index 3e092268a1..efe544e3cf 100644 --- a/core/consensus/babe/impl/block_appender_impl.cpp +++ b/core/consensus/babe/impl/block_appender_impl.cpp @@ -12,7 +12,7 @@ #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" #include "consensus/babe/types/slot.hpp" -#include "consensus/grandpa/impl/voting_round_error.hpp" +#include "consensus/grandpa/voting_round_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "primitives/common.hpp" #include "scale/scale.hpp" diff --git a/core/consensus/babe/impl/block_executor_impl.cpp b/core/consensus/babe/impl/block_executor_impl.cpp index a5506fac03..0a6de0f471 100644 --- a/core/consensus/babe/impl/block_executor_impl.cpp +++ b/core/consensus/babe/impl/block_executor_impl.cpp @@ -12,7 +12,7 @@ #include "consensus/babe/impl/babe_digests_util.hpp" #include "consensus/babe/impl/threshold_util.hpp" #include "consensus/babe/types/slot.hpp" -#include "consensus/grandpa/impl/voting_round_error.hpp" +#include "consensus/grandpa/voting_round_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "primitives/common.hpp" #include "runtime/runtime_api/offchain_worker_api.hpp" diff --git a/core/consensus/grandpa/CMakeLists.txt b/core/consensus/grandpa/CMakeLists.txt index 2304ee8c6b..2b3c742521 100644 --- a/core/consensus/grandpa/CMakeLists.txt +++ b/core/consensus/grandpa/CMakeLists.txt @@ -39,7 +39,7 @@ target_link_libraries(voter_set ) add_library(voting_round_error - impl/voting_round_error.cpp + voting_round_error.cpp ) target_link_libraries(voting_round_error outcome diff --git a/core/consensus/grandpa/impl/grandpa_impl.cpp b/core/consensus/grandpa/impl/grandpa_impl.cpp index 3d0f2a591f..8aa17aa3b2 100644 --- a/core/consensus/grandpa/impl/grandpa_impl.cpp +++ b/core/consensus/grandpa/impl/grandpa_impl.cpp @@ -8,9 +8,9 @@ #include "consensus/grandpa/grandpa_context.hpp" #include "consensus/grandpa/impl/vote_crypto_provider_impl.hpp" #include "consensus/grandpa/impl/vote_tracker_impl.hpp" -#include "consensus/grandpa/impl/voting_round_error.hpp" #include "consensus/grandpa/impl/voting_round_impl.hpp" #include "consensus/grandpa/vote_graph/vote_graph_impl.hpp" +#include "consensus/grandpa/voting_round_error.hpp" #include "network/helpers/peer_id_formatter.hpp" #include "scale/scale.hpp" diff --git a/core/consensus/grandpa/impl/voting_round_impl.cpp b/core/consensus/grandpa/impl/voting_round_impl.cpp index bc9ca498b4..8c2e429218 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.cpp +++ b/core/consensus/grandpa/impl/voting_round_impl.cpp @@ -11,7 +11,7 @@ #include "common/visitor.hpp" #include "consensus/grandpa/grandpa.hpp" #include "consensus/grandpa/grandpa_context.hpp" -#include "consensus/grandpa/impl/voting_round_error.hpp" +#include "consensus/grandpa/voting_round_error.hpp" namespace kagome::consensus::grandpa { @@ -1091,8 +1091,11 @@ namespace kagome::consensus::grandpa { BOOST_ASSERT(vote.is()); // Check if voter is contained in current voter set - OUTCOME_TRY(index_and_weight, voter_set_->indexAndWeight(vote.id)); - const auto &[index, weight] = index_and_weight; + auto index_and_weight_opt = voter_set_->indexAndWeight(vote.id); + if (!index_and_weight_opt) { + return VotingRoundError::UNKNOWN_VOTER; + } + const auto &[index, weight] = index_and_weight_opt.value(); auto [type, type_str_, equivocators, tracker] = [&]() -> std::tuple +#include "consensus/grandpa/voting_round_error.hpp" + namespace kagome::consensus::grandpa { namespace { @@ -90,8 +92,8 @@ namespace kagome::consensus::grandpa { const BlockInfo &block, const Id &voter) { auto inw_res = voter_set_->indexAndWeight(voter); - if (inw_res.has_error()) { - return inw_res.as_failure(); + if (!inw_res.has_value()) { + return VotingRoundError::UNKNOWN_VOTER; } const auto [index, weight] = inw_res.value(); diff --git a/core/consensus/grandpa/vote_graph/vote_graph_impl.hpp b/core/consensus/grandpa/vote_graph/vote_graph_impl.hpp index 4b7a4e3ffa..0c57b68973 100644 --- a/core/consensus/grandpa/vote_graph/vote_graph_impl.hpp +++ b/core/consensus/grandpa/vote_graph/vote_graph_impl.hpp @@ -109,6 +109,7 @@ namespace kagome::consensus::grandpa { BlockInfo base_; std::shared_ptr voter_set_; std::shared_ptr chain_; + log::Logger logger_; std::unordered_map entries_; std::unordered_set heads_; diff --git a/core/consensus/grandpa/voter_set.cpp b/core/consensus/grandpa/voter_set.cpp index 84330fb8e5..2ffa9cb133 100644 --- a/core/consensus/grandpa/voter_set.cpp +++ b/core/consensus/grandpa/voter_set.cpp @@ -10,12 +10,8 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::grandpa, VoterSet::Error, e) { switch (e) { case E::VOTER_ALREADY_EXISTS: return "Voter already exists"; - case E::VOTER_NOT_FOUND: - return "Voter not found"; case E::INDEX_OUTBOUND: return "Index outbound"; - case E::QUERYING_ZERO_VOTER: - return "Weight of a voter with a zero public key is queried"; } return "Unknown error (invalid VoterSet::Error)"; } @@ -49,14 +45,14 @@ namespace kagome::consensus::grandpa { return voter; } - outcome::result> + std::optional> VoterSet::indexAndWeight(const Id &voter) const { if (voter == Id{}) { - return Error::QUERYING_ZERO_VOTER; + return std::nullopt; } auto it = map_.find(voter); if (it == map_.end()) { - return Error::VOTER_NOT_FOUND; + return std::nullopt; } auto index = it->second; BOOST_ASSERT(index < list_.size()); @@ -64,26 +60,26 @@ namespace kagome::consensus::grandpa { return std::tuple(index, weight); } - outcome::result VoterSet::voterIndex(const Id &voter) const { + std::optional VoterSet::voterIndex(const Id &voter) const { if (voter == Id{}) { - return Error::QUERYING_ZERO_VOTER; + return std::nullopt; } auto it = map_.find(voter); if (it == map_.end()) { - return Error::VOTER_NOT_FOUND; + return std::nullopt; } auto index = it->second; return index; } - outcome::result VoterSet::voterWeight( + std::optional VoterSet::voterWeight( const Id &voter) const { if (voter == Id{}) { - return Error::QUERYING_ZERO_VOTER; + return std::nullopt; } auto it = map_.find(voter); if (it == map_.end()) { - return Error::VOTER_NOT_FOUND; + return std::nullopt; } auto index = it->second; BOOST_ASSERT(index < list_.size()); diff --git a/core/consensus/grandpa/voter_set.hpp b/core/consensus/grandpa/voter_set.hpp index 2b95a6604b..b5d325173c 100644 --- a/core/consensus/grandpa/voter_set.hpp +++ b/core/consensus/grandpa/voter_set.hpp @@ -20,9 +20,7 @@ namespace kagome::consensus::grandpa { public: enum class Error { VOTER_ALREADY_EXISTS = 1, - VOTER_NOT_FOUND, - INDEX_OUTBOUND, - QUERYING_ZERO_VOTER + INDEX_OUTBOUND }; using Index = size_t; @@ -44,7 +42,7 @@ namespace kagome::consensus::grandpa { return id_; } - outcome::result> indexAndWeight( + std::optional> indexAndWeight( const Id &voter) const; outcome::result voterId(Index index) const; @@ -52,12 +50,12 @@ namespace kagome::consensus::grandpa { /** * \return index of \param voter */ - outcome::result voterIndex(const Id &voter) const; + std::optional voterIndex(const Id &voter) const; /** * \return weight of \param voter */ - outcome::result voterWeight(const Id &voter) const; + std::optional voterWeight(const Id &voter) const; /** * \return weight of voter by index \param voter_index diff --git a/core/consensus/grandpa/impl/voting_round_error.cpp b/core/consensus/grandpa/voting_round_error.cpp similarity index 97% rename from core/consensus/grandpa/impl/voting_round_error.cpp rename to core/consensus/grandpa/voting_round_error.cpp index 9d475f4a68..45d1e70c4d 100644 --- a/core/consensus/grandpa/impl/voting_round_error.cpp +++ b/core/consensus/grandpa/voting_round_error.cpp @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "consensus/grandpa/impl/voting_round_error.hpp" +#include "voting_round_error.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::grandpa, VotingRoundError, e) { using E = kagome::consensus::grandpa::VotingRoundError; diff --git a/core/consensus/grandpa/impl/voting_round_error.hpp b/core/consensus/grandpa/voting_round_error.hpp similarity index 96% rename from core/consensus/grandpa/impl/voting_round_error.hpp rename to core/consensus/grandpa/voting_round_error.hpp index b45780b0a6..ea2e9db028 100644 --- a/core/consensus/grandpa/impl/voting_round_error.hpp +++ b/core/consensus/grandpa/voting_round_error.hpp @@ -6,7 +6,7 @@ #ifndef KAGOME_CORE_CONSENSUS_GRANDPA_IMPL_VOTING_ROUND_ERROR_HPP #define KAGOME_CORE_CONSENSUS_GRANDPA_IMPL_VOTING_ROUND_ERROR_HPP -#include +#include "outcome/outcome.hpp" namespace kagome::consensus::grandpa { diff --git a/housekeeping/indocker.sh b/housekeeping/indocker.sh index 7a7a89bc80..0131f63e72 100755 --- a/housekeeping/indocker.sh +++ b/housekeeping/indocker.sh @@ -18,7 +18,7 @@ docker build -t soramitsu/kagome:local-dev $DIR rm $DIR/Dockerfile -docker run -i --rm \ +docker run -it --rm \ --cap-add SYS_PTRACE \ -v /tmp/cache/hunter:/root/.hunter \ -w /workdir \ @@ -26,4 +26,5 @@ docker run -i --rm \ -e CODECOV_TOKEN \ -e SONAR_TOKEN \ $CI_ENV \ - soramitsu/kagome:local-dev + soramitsu/kagome:local-dev \ + /bin/bash diff --git a/test/core/consensus/babe/babe_test.cpp b/test/core/consensus/babe/babe_test.cpp index 620d0df81a..0072f5bfb7 100644 --- a/test/core/consensus/babe/babe_test.cpp +++ b/test/core/consensus/babe/babe_test.cpp @@ -21,6 +21,7 @@ #include "mock/core/consensus/authority/authority_update_observer_mock.hpp" #include "mock/core/consensus/babe/babe_util_mock.hpp" #include "mock/core/consensus/babe/block_executor_mock.hpp" +#include "mock/core/consensus/babe/consistency_keeper_mock.hpp" #include "mock/core/consensus/babe_lottery_mock.hpp" #include "mock/core/consensus/grandpa/environment_mock.hpp" #include "mock/core/consensus/validation/block_validator_mock.hpp" @@ -145,6 +146,8 @@ class BabeTest : public testing::Test { EXPECT_CALL(*sr25519_provider, sign(_, _)) .WillRepeatedly(Return(Sr25519Signature{})); + consistency_keeper_ = std::make_shared(); + babe_ = std::make_shared(app_config_, app_state_manager_, lottery_, @@ -160,7 +163,8 @@ class BabeTest : public testing::Test { grandpa_authority_update_observer_, synchronizer_, babe_util_, - offchain_worker_api_); + offchain_worker_api_, + consistency_keeper_); epoch_.start_slot = 0; epoch_.epoch_number = 0; @@ -198,6 +202,7 @@ class BabeTest : public testing::Test { std::shared_ptr babe_config_; std::shared_ptr babe_util_; std::shared_ptr offchain_worker_api_; + std::shared_ptr consistency_keeper_; std::shared_ptr io_context_; std::shared_ptr babe_; diff --git a/test/core/consensus/grandpa/voter_set_test.cpp b/test/core/consensus/grandpa/voter_set_test.cpp index 0fce568c33..ee0b0a29f5 100644 --- a/test/core/consensus/grandpa/voter_set_test.cpp +++ b/test/core/consensus/grandpa/voter_set_test.cpp @@ -66,12 +66,11 @@ TEST_F(VoterSetTest, GetIndex) { for (auto &[voter, weight] : voters) { // WHEN+THEN.1 - EXPECT_OUTCOME_SUCCESS(index, testee->voterIndex(voter)); + ASSERT_NE(std::nullopt, testee->voterIndex(voter)); } // WHEN+THEN.2 - EXPECT_OUTCOME_ERROR( - res, testee->voterIndex("Unknown"_ID), VoterSet::Error::VOTER_NOT_FOUND); + ASSERT_EQ(testee->voterIndex("Unknown"_ID), std::nullopt); } TEST_F(VoterSetTest, GetWeight) { @@ -85,8 +84,9 @@ TEST_F(VoterSetTest, GetWeight) { for (auto &[voter, weight] : voters) { { // WHEN+THEN.1 - EXPECT_OUTCOME_SUCCESS(actual_weight_res, testee->voterWeight(voter)); - auto &actual_weight = actual_weight_res.value(); + auto actual_weight_opt = testee->voterWeight(voter); + ASSERT_TRUE(actual_weight_opt.has_value()); + auto &actual_weight = actual_weight_opt.value(); EXPECT_EQ(weight, actual_weight); } { @@ -100,9 +100,7 @@ TEST_F(VoterSetTest, GetWeight) { { // WHEN+THEN.3 - EXPECT_OUTCOME_ERROR(res, - testee->voterWeight("Unknown"_ID), - VoterSet::Error::VOTER_NOT_FOUND); + ASSERT_EQ(std::nullopt, testee->voterWeight("Unknown"_ID)); } { // WHEN+THEN.4 @@ -141,8 +139,9 @@ TEST_F(VoterSetTest, GetIndexAndWeight) { size_t index = 0; for (auto &[voter, weight] : voters) { // WHEN+THEN.1 - ASSERT_OUTCOME_SUCCESS(res, testee->indexAndWeight(voter)); - auto [actual_index, actual_weight] = res; + auto res = testee->indexAndWeight(voter); + ASSERT_NE(res, std::nullopt); + auto [actual_index, actual_weight] = res.value(); EXPECT_TRUE(actual_index == index); EXPECT_TRUE(actual_weight == weight); ++index; @@ -150,8 +149,6 @@ TEST_F(VoterSetTest, GetIndexAndWeight) { { // WHEN+THEN.2 - EXPECT_OUTCOME_ERROR(res, - testee->indexAndWeight("Unknown"_ID), - VoterSet::Error::VOTER_NOT_FOUND); + ASSERT_EQ(std::nullopt, testee->indexAndWeight("Unknown"_ID)); } } diff --git a/test/core/consensus/grandpa/voting_round/voting_round_test.cpp b/test/core/consensus/grandpa/voting_round/voting_round_test.cpp index cda63ac636..ae127b9a85 100644 --- a/test/core/consensus/grandpa/voting_round/voting_round_test.cpp +++ b/test/core/consensus/grandpa/voting_round/voting_round_test.cpp @@ -10,9 +10,9 @@ #include "clock/impl/clock_impl.hpp" #include "common/visitor.hpp" #include "consensus/grandpa/impl/vote_tracker_impl.hpp" -#include "consensus/grandpa/impl/voting_round_error.hpp" #include "consensus/grandpa/impl/voting_round_impl.hpp" #include "consensus/grandpa/vote_graph/vote_graph_impl.hpp" +#include "consensus/grandpa/voting_round_error.hpp" #include "core/consensus/grandpa/literals.hpp" #include "mock/core/consensus/authority/authority_manager_mock.hpp" #include "mock/core/consensus/grandpa/environment_mock.hpp" From 7894fc8550ad038e80984b04f94e5424b84b6f9e Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 3 Oct 2022 18:16:01 +0300 Subject: [PATCH 63/74] Fixes for fast sync in kusama --- .../authority/impl/authority_manager_impl.cpp | 662 ++++++++++++++---- .../authority/impl/authority_manager_impl.hpp | 22 +- .../babe/impl/consistency_keeper_impl.cpp | 2 +- .../babe/types/babe_block_header.hpp | 2 +- core/consensus/grandpa/impl/grandpa_impl.cpp | 9 +- .../grandpa/impl/voting_round_impl.cpp | 8 +- core/consensus/grandpa/voting_round_error.cpp | 2 +- .../validation/babe_block_validator.cpp | 18 +- core/network/impl/stream_engine.hpp | 2 +- 9 files changed, 563 insertions(+), 164 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index a3ad32d759..d8983b08f4 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -30,6 +30,81 @@ using kagome::primitives::AuthoritySetId; namespace kagome::authority { + struct ScheduledChangeEntry { + primitives::AuthoritySet new_authorities; + primitives::BlockInfo scheduled_at; + primitives::BlockNumber activates_at; + }; + + template + struct ForkTree { + T payload; + primitives::BlockInfo block; + std::vector> children; + ForkTree *parent; + + ForkTree *find(primitives::BlockHash const &block_hash) const { + if (block.hash == block_hash) return this; + for (auto &child : children) { + if (auto *res = child->find(block_hash); res != nullptr) { + return res; + } + } + return nullptr; + } + + bool forEachLeaf(std::function const &f) { + if (children.empty()) { + return f(*this); + } + for (auto &child : children) { + bool res = child->forEachLeaf(f); + if (res) return true; + } + return false; + } + + std::unique_ptr> detachChild(ForkTree &child) { + for (size_t i = 0; i < children.size(); i++) { + auto ¤t_child = children[i]; + if (current_child.get() == &child) { + auto owned_child = std::move(current_child); + owned_child->parent = nullptr; + children.erase(children.begin() + i); + return owned_child; + } + } + return nullptr; + } + }; + + ::scale::ScaleEncoderStream &operator<<(::scale::ScaleEncoderStream &s, + const ScheduledChangeEntry &entry) { + s << entry.scheduled_at << entry.activates_at << entry.new_authorities; + return s; + } + + ::scale::ScaleDecoderStream &operator>>(::scale::ScaleDecoderStream &s, + ScheduledChangeEntry &entry) { + s >> entry.scheduled_at >> entry.activates_at >> entry.new_authorities; + return s; + } + + ::scale::ScaleEncoderStream &operator<<(::scale::ScaleEncoderStream &s, + const ScheduleTree &tree) { + s << tree.block << tree.payload << tree.children; + return s; + } + + ::scale::ScaleDecoderStream &operator>>(::scale::ScaleDecoderStream &s, + ScheduleTree &change) { + s >> change.block >> change.payload >> change.children; + for (auto &child : change.children) { + child->parent = &change; + } + return s; + } + AuthorityManagerImpl::AuthorityManagerImpl( Config config, std::shared_ptr app_state_manager, @@ -243,15 +318,62 @@ namespace kagome::authority { BOOST_UNREACHABLE_RETURN({}) } + static const Buffer storage_key = + Buffer::fromString("test_authority_manager_root"); + + AuthorityManagerImpl::~AuthorityManagerImpl() {} + bool AuthorityManagerImpl::prepare() { const auto finalized_block = block_tree_->getLastFinalized(); - auto res = initializeAt(finalized_block); + + auto res = persistent_storage_->tryLoad(storage_key); if (!res) { SL_ERROR(log_, - "Error initializing authority manager: {}", + "Error loading authority manager root from DB: {}", res.error().message()); + return false; + } + auto &root_opt = res.value(); + if (root_opt) { + auto [current_set, scheduled_changes] = + scale::decode>>( + root_opt.value()) + .value(); + scheduled_changes_ = std::move(scheduled_changes); + current_set_ = std::move(current_set); + current_block_ = scheduled_changes_->block; + } else if (finalized_block.number == 0) { + primitives::BlockInfo genesis_block_info{ + block_tree_->getGenesisBlockHash(), 0}; + scheduled_changes_ = std::make_unique( + ScheduleTree{.payload{ScheduledChangeEntry{ + .new_authorities{primitives::AuthoritySet( + 0, grandpa_api_->authorities(0).value())}, + .scheduled_at{primitives::BlockInfo{ + 0, block_tree_->getGenesisBlockHash()}}, + .activates_at = 0}}, + .block = genesis_block_info, + .children = {}, + .parent = nullptr}); + } else { + SL_ERROR( + log_, + "Failed to initialize authority manager; Try recovering the state."); + return false; } - return res.has_value(); + SL_DEBUG(log_, "Initialized authority manager at block {} with set id {}", + current_block_, + current_set_.id); + return true; + // + // auto res = initializeAt(finalized_block); + // if (!res) { + // SL_ERROR(log_, + // "Error initializing authority manager: {}", + // res.error().message()); + // } + // return res.has_value(); } outcome::result AuthorityManagerImpl::initializeAt( @@ -391,10 +513,15 @@ namespace kagome::authority { auto genesis_hash = block_tree_->getGenesisBlockHash(); OUTCOME_TRY(initial_authorities, grandpa_api_->authorities(genesis_hash)); - - root_ = ScheduleNode::createAsRoot( - std::make_shared(0, initial_authorities), - {0, genesis_hash}); + primitives::BlockInfo genesis_info{0, block_tree_->getGenesisBlockHash()}; + scheduled_changes_ = std::make_unique(ScheduleTree{ + .payload{ScheduledChangeEntry{ + .new_authorities{primitives::AuthoritySet(0, initial_authorities)}, + .scheduled_at{genesis_info}, + .activates_at = 0}}, + .block = genesis_info, + .children = {}, + .parent = nullptr}); SL_INFO(log_, "Recovering authority manager state... (might take a few minutes)"); // if state is pruned @@ -437,7 +564,59 @@ namespace kagome::authority { start = end; } } + return outcome::success(); + // + // root_ = ScheduleNode::createAsRoot( + // std::make_shared(0, + // initial_authorities), {0, genesis_hash}); + // SL_INFO(log_, + // "Recovering authority manager state... (might take a few + // minutes)"); + // // if state is pruned + // if (header_repo_->getBlockHeader(1).has_error()) { + // SL_WARN(log_, + // "Can't recalculate authority set id on a prune state, fall + // back " "to fetching from runtime"); + // return clearScheduleGraphRoot(*persistent_storage_); + // } + // + // auto start = std::chrono::steady_clock::now(); + // for (primitives::BlockNumber number = 0; number <= + // last_finalized_number; + // number++) { + // auto header_res = header_repo_->getBlockHeader(number); + // if (!header_res) + // continue; // Temporary workaround about the justification pruning + // bug + // auto &header = header_res.value(); + // + // OUTCOME_TRY(hash, header_repo_->getHashByNumber(number)); + // primitives::BlockInfo info{number, hash}; + // + // for (auto &msg : header.digest) { + // if (auto consensus_msg = boost::get(&msg); + // consensus_msg != nullptr) { + // onConsensus(info, *consensus_msg).value(); + // } + // } + // auto justification_res = block_tree_->getBlockJustification(hash); + // if (justification_res.has_value()) prune(info); + // auto end = std::chrono::steady_clock::now(); + // auto duration = end - start; + // using namespace std::chrono_literals; + // // 5 seconds is nothing special, just a random more-or-like + // convenient + // // duration. + // if (duration > 5s) { + // SL_VERBOSE(log_, + // "Processed {} out of {} blocks", + // number, + // last_finalized_number); + // start = end; + // } + // } + // return outcome::success(); } primitives::BlockInfo AuthorityManagerImpl::base() const { @@ -451,139 +630,226 @@ namespace kagome::authority { std::optional> AuthorityManagerImpl::authorities(const primitives::BlockInfo &target_block, IsBlockFinalized finalized) const { - auto node = getAppropriateAncestor(target_block); - - if (node == nullptr) { - return std::nullopt; + auto ancestor = findClosestAncestor(*scheduled_changes_, target_block); + assert(ancestor->block.number <= target_block.number); + if (ancestor == nullptr) return std::nullopt; + if (ancestor->payload.activates_at <= target_block.number) { + return std::make_shared( + ancestor->payload.new_authorities); } - - IsBlockFinalized node_in_finalized_chain = - node->current_block == target_block - ? (bool)finalized - : node->current_block.number - <= block_tree_->getLastFinalized().number; - - auto adjusted_node = - node->makeDescendant(target_block, node_in_finalized_chain); - - if (adjusted_node->enabled) { - // Original authorities - SL_DEBUG(log_, - "Pick authority set with id {} for block {}", - adjusted_node->current_authorities->id, - target_block); - // Since getAppropriateAncestor worked normally on this block - auto header = block_tree_->getBlockHeader(target_block.hash).value(); - if (auto id_from_storage_res = readSetIdFromRuntime(header); - id_from_storage_res) { - auto &id_from_storage_opt = id_from_storage_res.value(); - if (id_from_storage_opt) { - SL_DEBUG(log_, - "Pick authority set id from runtime: {}", - id_from_storage_opt.value()); - } else { - SL_DEBUG(log_, - "Failed to pick authority set id from runtime: not found in " - "trie storage"); - } - } else { - SL_DEBUG(log_, - "Failed to obtain authority set id from runtime: ", - id_from_storage_res.error().message()); - } - for (auto& authority: adjusted_node->current_authorities->authorities) { - SL_TRACE(log_, "Authority {}: {}", authority.id.id, authority.weight); - } - return adjusted_node->current_authorities; + if (ancestor->parent == scheduled_changes_.get()) { + return std::make_shared(current_set_); } - - // Zero-weighted authorities - auto authorities = std::make_shared( - *adjusted_node->current_authorities); - std::for_each(authorities->begin(), - authorities->end(), - [](auto &authority) { authority.weight = 0; }); - return authorities; + BOOST_ASSERT(ancestor->parent != nullptr); + return std::make_shared( + ancestor->parent->payload.new_authorities); + // + // auto node = getAppropriateAncestor(target_block); + // + // if (node == nullptr) { + // return std::nullopt; + // } + // + // IsBlockFinalized node_in_finalized_chain = + // node->current_block == target_block + // ? (bool)finalized + // : node->current_block.number + // <= block_tree_->getLastFinalized().number; + // + // auto adjusted_node = + // node->makeDescendant(target_block, node_in_finalized_chain); + // + // if (adjusted_node->enabled) { + // // Original authorities + // SL_DEBUG(log_, + // "Pick authority set with id {} for block {}", + // adjusted_node->current_authorities->id, + // target_block); + // // Since getAppropriateAncestor worked normally on this block + // auto header = + // block_tree_->getBlockHeader(target_block.hash).value(); if (auto + // id_from_storage_res = readSetIdFromRuntime(header); + // id_from_storage_res) { + // auto &id_from_storage_opt = id_from_storage_res.value(); + // if (id_from_storage_opt) { + // SL_DEBUG(log_, + // "Pick authority set id from runtime: {}", + // id_from_storage_opt.value()); + // } else { + // SL_DEBUG(log_, + // "Failed to pick authority set id from runtime: not + // found in " "trie storage"); + // } + // } else { + // SL_DEBUG(log_, + // "Failed to obtain authority set id from runtime: ", + // id_from_storage_res.error().message()); + // } + // for (auto &authority : + // adjusted_node->current_authorities->authorities) { + // SL_TRACE(log_, "Authority {}: {}", authority.id.id, + // authority.weight); + // } + // return adjusted_node->current_authorities; + // } + // + // // Zero-weighted authorities + // auto authorities = std::make_shared( + // *adjusted_node->current_authorities); + // std::for_each(authorities->begin(), + // authorities->end(), + // [](auto &authority) { authority.weight = 0; }); + // return authorities; } outcome::result AuthorityManagerImpl::applyScheduledChange( const primitives::BlockInfo &block, const primitives::AuthorityList &authorities, primitives::BlockNumber activate_at) { - SL_DEBUG(log_, - "Applying scheduled change on block {} to activate at block {}", - block, - activate_at); - KAGOME_PROFILE_START(get_appropriate_ancestor) - auto ancestor_node = getAppropriateAncestor(block); - KAGOME_PROFILE_END(get_appropriate_ancestor) - - if (not ancestor_node) { + if (activate_at < current_block_.number) { + SL_ERROR(log_, + "Error: scheduling changes for a block #{} earlier than the " + "current root #{}", + activate_at, + current_block_.number); return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; } - SL_DEBUG(log_, - "Authorities for block {} found on block {} with set id {}", - block, - ancestor_node->current_block, - ancestor_node->current_authorities->id); - - auto schedule_change = [&](const std::shared_ptr &node) - -> outcome::result { - auto new_authorities = std::make_shared( - node->current_authorities->id + 1, authorities); + ForkTree *nearest_change_node = nullptr; + scheduled_changes_->forEachLeaf( + [this, &block, &nearest_change_node](auto &leaf) { + if (leaf.block.number <= block.number + && block_tree_->hasDirectChain(leaf.block.hash, block.hash)) { + nearest_change_node = &leaf; + return true; + } + return false; + }); - node->action = - ScheduleNode::ScheduledChange{activate_at, new_authorities}; + if (nearest_change_node == nullptr) { + SL_ERROR(log_, + "Neither of pending scheduled changes could be the ancestor of " + "the proposed one (scheduled at #{}).", + block.number); + return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; + } - SL_VERBOSE( + auto &nearest_change = nearest_change_node->payload; + // the earliest change encountered in a block is respected + if (nearest_change.scheduled_at.number == block.number) { + SL_DEBUG(log_, + "Ignore scheduled change at {} as this block already " + "schedules a change", + block); + } else if (nearest_change_node->payload.activates_at >= block.number) { + SL_DEBUG( log_, - "Authority set change is scheduled after block #{} (set id={})", - activate_at, - new_authorities->id); - - size_t index = 0; - for (auto &authority : *new_authorities) { - SL_DEBUG(log_, - "New authority ({}/{}): id={} weight={}", - ++index, - new_authorities->authorities.size(), - authority.id.id, - authority.weight); - } - - return outcome::success(); - }; - - /// TODO(Harrm): Should account for finality when fetching an authority set - /// for some purposes, but not when scheduling further changes - IsBlockFinalized is_ancestor_node_finalized = true; - - if (ancestor_node->current_block == block) { - ancestor_node->adjust(is_ancestor_node_finalized); - - OUTCOME_TRY(schedule_change(ancestor_node)); + "Ignore scheduled change at {} as the previous scheduled change " + "is not applied yet", + block); } else { - KAGOME_PROFILE_START(make_descendant) - auto new_node = - ancestor_node->makeDescendant(block, is_ancestor_node_finalized); - KAGOME_PROFILE_END(make_descendant) SL_DEBUG(log_, - "Make a schedule node for block {}, with actual set id {}", + "Applying scheduled change with id {} on block {} to activate " + "at block {}", + nearest_change.new_authorities.id + 1, block, - new_node->current_authorities->id); - - KAGOME_PROFILE_START(schedule_change) - OUTCOME_TRY(schedule_change(new_node)); - KAGOME_PROFILE_END(schedule_change) + activate_at); + size_t i = 0; + SL_TRACE(log_, "Authorities:"); + for(auto& authority: authorities) { + SL_TRACE(log_, "{} - {}", i++, authority.id.id.toHex()); + } - // Reorganize ancestry - KAGOME_PROFILE_START(reorganize) - reorganize(ancestor_node, new_node); - KAGOME_PROFILE_END(reorganize) + nearest_change_node->children.emplace_back( + std::unique_ptr>{ + new ForkTree{ + .payload = + ScheduledChangeEntry{ + .new_authorities = + primitives::AuthoritySet{ + .id = nearest_change.new_authorities.id + 1, + .authorities = authorities, + }, + .scheduled_at = block, + .activates_at = activate_at}, + .block = block, + .children{}, + .parent = nearest_change_node}}); } - return outcome::success(); + // + // KAGOME_PROFILE_START(get_appropriate_ancestor) + // auto ancestor_node = getAppropriateAncestor(block); + // KAGOME_PROFILE_END(get_appropriate_ancestor) + // + // if (not ancestor_node) { + // return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; + // } + // + // SL_DEBUG(log_, + // "Authorities for block {} found on block {} with set id {}", + // block, + // ancestor_node->current_block, + // ancestor_node->current_authorities->id); + // + // auto schedule_change = [&](const std::shared_ptr &node) + // -> outcome::result { + // auto new_authorities = std::make_shared( + // node->current_authorities->id + 1, authorities); + // + // node->action = + // ScheduleNode::ScheduledChange{activate_at, new_authorities}; + // + // SL_VERBOSE( + // log_, + // "Authority set change is scheduled after block #{} (set id={})", + // activate_at, + // new_authorities->id); + // + // size_t index = 0; + // for (auto &authority : *new_authorities) { + // SL_DEBUG(log_, + // "New authority ({}/{}): id={} weight={}", + // ++index, + // new_authorities->authorities.size(), + // authority.id.id, + // authority.weight); + // } + // + // return outcome::success(); + // }; + // + // /// TODO(Harrm): Should account for finality when fetching an + // authority set + // /// for some purposes, but not when scheduling further changes + // IsBlockFinalized is_ancestor_node_finalized = true; + // + // if (ancestor_node->current_block == block) { + // ancestor_node->adjust(is_ancestor_node_finalized); + // + // OUTCOME_TRY(schedule_change(ancestor_node)); + // } else { + // KAGOME_PROFILE_START(make_descendant) + // auto new_node = + // ancestor_node->makeDescendant(block, + // is_ancestor_node_finalized); + // KAGOME_PROFILE_END(make_descendant) + // SL_DEBUG(log_, + // "Make a schedule node for block {}, with actual set id {}", + // block, + // new_node->current_authorities->id); + // + // KAGOME_PROFILE_START(schedule_change) + // OUTCOME_TRY(schedule_change(new_node)); + // KAGOME_PROFILE_END(schedule_change) + // + // // Reorganize ancestry + // KAGOME_PROFILE_START(reorganize) + // reorganize(ancestor_node, new_node); + // KAGOME_PROFILE_END(reorganize) + // } + // + // return outcome::success(); } outcome::result AuthorityManagerImpl::applyForcedChange( @@ -807,17 +1073,26 @@ namespace kagome::authority { block, msg.authorities, block.number + msg.subchain_length); }, [this, &block](const primitives::ForcedChange &msg) { + throw std::runtime_error{"not implemented"}; return applyForcedChange( block, msg.authorities, msg.delay_start, msg.subchain_length); }, [this, &block](const primitives::OnDisabled &msg) { - return applyOnDisabled(block, msg.authority_index); + SL_DEBUG(log_, "OnDisabled {}", msg.authority_index); + // return applyOnDisabled(block, msg.authority_index); + return outcome::success(); }, [this, &block](const primitives::Pause &msg) { - return applyPause(block, block.number + msg.subchain_length); + SL_DEBUG(log_, "Pause {}", msg.subchain_length); + // return applyPause(block, block.number + + // msg.subchain_length); + return outcome::success(); }, [this, &block](const primitives::Resume &msg) { - return applyResume(block, block.number + msg.subchain_length); + SL_DEBUG(log_, "Resume {}", msg.subchain_length); + // return applyResume(block, block.number + + // msg.subchain_length); + return outcome::success(); }, [](auto &) { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; @@ -838,6 +1113,49 @@ namespace kagome::authority { } void AuthorityManagerImpl::prune(const primitives::BlockInfo &block) { + if (block.number <= scheduled_changes_->block.number) { + return; + } + auto ancestor = findClosestAncestor(*scheduled_changes_, block); + if (ancestor == scheduled_changes_.get()) { + if (scheduled_changes_->block.number + < scheduled_changes_->payload.activates_at + && block.number >= scheduled_changes_->payload.activates_at) + current_set_ = scheduled_changes_->payload.new_authorities; + scheduled_changes_->block = block; + current_block_ = block; + } + if (ancestor->block < block) { + ancestor->block = block; + } + + if (ancestor->payload.activates_at > block.number) { + if (ancestor->parent != nullptr) { + current_set_ = ancestor->parent->payload.new_authorities; + } else { + assert(ancestor == scheduled_changes_.get()); + // current set remains unchanged + } + } else { + current_set_ = ancestor->payload.new_authorities; + } + + std::unique_ptr new_root{}; + if (ancestor->parent != nullptr) { + new_root = ancestor->parent->detachChild(*ancestor); + } else { + assert(ancestor == scheduled_changes_.get()); + new_root = std::move(scheduled_changes_); + } + assert(new_root->parent == nullptr); + current_block_ = new_root->block; + scheduled_changes_ = std::move(new_root); + persistent_storage_ + ->put(storage_key, + Buffer{scale::encode(current_set_, scheduled_changes_).value()}) + .value(); + return; + if (block == root_->current_block) { return; } @@ -952,41 +1270,93 @@ namespace kagome::authority { } void AuthorityManagerImpl::cancel(const primitives::BlockInfo &block) { - auto ancestor = getAppropriateAncestor(block); - + auto ancestor = findClosestAncestor(*scheduled_changes_, block); if (ancestor == nullptr) { - SL_TRACE(log_, "No scheduled changes on block {}: no ancestor", block); + SL_TRACE(log_, + "Can't cancel scheduled changes for {}, block is either " + "finalized or unknown", + block); return; } - - if (ancestor == root_) { - // Can't remove root + assert(ancestor->block.number <= block.number); + if (ancestor == scheduled_changes_.get()) { SL_TRACE(log_, - "Can't cancel scheduled changes on block {}: it is root", + "Can't cancel scheduled changes for {}, block is the finalized " + "root of schedule changes tree", block); return; } - - if (ancestor->current_block == block) { - ancestor = std::const_pointer_cast(ancestor->parent.lock()); + assert(ancestor->parent != nullptr); + if (ancestor->block == block) { + ancestor->parent->detachChild(*ancestor); + return; + } + for (auto &child : ancestor->children) { + if (block_tree_->hasDirectChain(block.hash, child->block.hash)) { + ancestor->detachChild(*child); + } } - auto it = std::find_if(ancestor->descendants.begin(), - ancestor->descendants.end(), - [&block](std::shared_ptr node) { - return node->current_block == block; - }); + // auto ancestor = getAppropriateAncestor(block); + // + // if (ancestor == nullptr) { + // SL_TRACE(log_, "No scheduled changes on block {}: no ancestor", + // block); return; + // } + // + // if (ancestor == root_) { + // // Can't remove root + // SL_TRACE(log_, + // "Can't cancel scheduled changes on block {}: it is root", + // block); + // return; + // } + // + // if (ancestor->current_block == block) { + // ancestor = + // std::const_pointer_cast(ancestor->parent.lock()); + // } + // + // auto it = std::find_if(ancestor->descendants.begin(), + // ancestor->descendants.end(), + // [&block](std::shared_ptr node) { + // return node->current_block == block; + // }); + // + // if (it != ancestor->descendants.end()) { + // if (not(*it)->descendants.empty()) { + // // Has descendants - is not a leaf + // SL_TRACE(log_, "No scheduled changes on block {}: not found", + // block); return; + // } + // + // SL_DEBUG(log_, "Scheduled changes on block {} has removed", block); + // ancestor->descendants.erase(it); + // } + } - if (it != ancestor->descendants.end()) { - if (not(*it)->descendants.empty()) { - // Has descendants - is not a leaf - SL_TRACE(log_, "No scheduled changes on block {}: not found", block); - return; - } + ScheduleTree *AuthorityManagerImpl::findClosestAncestor( + ForkTree ¤t, + primitives::BlockInfo const &block) const { + auto *res = findClosestAncestor( + const_cast const &>(current), block); + return const_cast *>(res); + } - SL_DEBUG(log_, "Scheduled changes on block {} has removed", block); - ancestor->descendants.erase(it); + ScheduleTree const *AuthorityManagerImpl::findClosestAncestor( + ForkTree const ¤t, + primitives::BlockInfo const &block) const { + for (auto &child : current.children) { + if (child->block.number < block.number + && block_tree_->hasDirectChain(child->block.hash, block.hash)) { + return findClosestAncestor(*child, block); + } + } + if (current.block.number <= block.number + && block_tree_->hasDirectChain(current.block.hash, block.hash)) { + return ¤t; } + return nullptr; } } // namespace kagome::authority diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 05d527f8f1..36d7269974 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -39,6 +39,11 @@ namespace kagome::storage::trie { } namespace kagome::authority { + + struct ScheduledChangeEntry; + template struct ForkTree; + using ScheduleTree = ForkTree; + class AuthorityManagerImpl : public AuthorityManager, public AuthorityUpdateObserver { public: @@ -63,7 +68,7 @@ namespace kagome::authority { std::shared_ptr persistent_storage, std::shared_ptr header_repo); - ~AuthorityManagerImpl() override = default; + ~AuthorityManagerImpl() override; outcome::result recalculateStoredState( primitives::BlockNumber last_finalized_number) override; @@ -132,8 +137,21 @@ namespace kagome::authority { void reorganize(std::shared_ptr node, std::shared_ptr new_node); + + ScheduleTree *findClosestAncestor( + ScheduleTree ¤t, + primitives::BlockInfo const &block) const; + + ScheduleTree const *findClosestAncestor( + ScheduleTree const ¤t, + primitives::BlockInfo const &block) const; + + std::unique_ptr scheduled_changes_; + primitives::AuthoritySet current_set_; + primitives::BlockInfo current_block_; + Config config_; - std::shared_ptr block_tree_; + std::shared_ptr block_tree_; std::shared_ptr trie_storage_; std::shared_ptr grandpa_api_; std::shared_ptr hasher_; diff --git a/core/consensus/babe/impl/consistency_keeper_impl.cpp b/core/consensus/babe/impl/consistency_keeper_impl.cpp index 7164e8bce5..7240e85503 100644 --- a/core/consensus/babe/impl/consistency_keeper_impl.cpp +++ b/core/consensus/babe/impl/consistency_keeper_impl.cpp @@ -55,7 +55,7 @@ namespace kagome::consensus::babe { auto &block = block_res.value(); SL_WARN(logger_, - "Found partial applied block {}. Trying to rollback him", + "Found partial applied block {}. Trying to rollback it", block); rollback(block); diff --git a/core/consensus/babe/types/babe_block_header.hpp b/core/consensus/babe/types/babe_block_header.hpp index 43e708931c..70118b8cdc 100644 --- a/core/consensus/babe/types/babe_block_header.hpp +++ b/core/consensus/babe/types/babe_block_header.hpp @@ -44,7 +44,7 @@ namespace kagome::consensus { return slot_assignment_type == SlotType::Primary; } - bool needAuthorCheck() const { + bool isProducedInSecondarySlot() const { return slot_assignment_type == SlotType::SecondaryPlain or slot_assignment_type == SlotType::SecondaryVRF; } diff --git a/core/consensus/grandpa/impl/grandpa_impl.cpp b/core/consensus/grandpa/impl/grandpa_impl.cpp index 8aa17aa3b2..d3d015a72c 100644 --- a/core/consensus/grandpa/impl/grandpa_impl.cpp +++ b/core/consensus/grandpa/impl/grandpa_impl.cpp @@ -853,13 +853,16 @@ namespace kagome::consensus::grandpa { return VotingRoundError::JUSTIFICATION_FOR_AUTHORITY_SET_IN_PAST; } if (authority_set->id == current_round_->voterSetId() - && justification.round_number - < current_round_->roundNumber()) { + && justification.round_number < current_round_->roundNumber()) { return VotingRoundError::JUSTIFICATION_FOR_ROUND_IN_PAST; } if (authority_set->id > current_round_->voterSetId() + 1) { - return VotingRoundError::WRONG_ORDER_OF_VOTER_SET_ID; + SL_WARN(logger_, + "Authority set on block {} with justification has id {}, " + "while the current round set id is {} (difference must be 1)", + block_info, authority_set->id, current_round_->voterSetId()); + //return VotingRoundError::WRONG_ORDER_OF_VOTER_SET_ID; } auto voters = std::make_shared(authority_set->id); diff --git a/core/consensus/grandpa/impl/voting_round_impl.cpp b/core/consensus/grandpa/impl/voting_round_impl.cpp index 8c2e429218..af284f38c4 100644 --- a/core/consensus/grandpa/impl/voting_round_impl.cpp +++ b/core/consensus/grandpa/impl/voting_round_impl.cpp @@ -782,8 +782,13 @@ namespace kagome::consensus::grandpa { signed_precommit.id, signed_precommit.getBlockHash()); success) { // New vote + auto weight_opt = voter_set_->voterWeight(signed_precommit.id); + if (!weight_opt) { + SL_DEBUG(logger_, "Voter {} is not in the current voter set", signed_precommit.id.toHex()); + continue; + } if (env_->hasAncestry(vote.hash, signed_precommit.getBlockHash())) { - total_weight += voter_set_->voterWeight(signed_precommit.id).value(); + total_weight += weight_opt.value(); } else { SL_DEBUG(logger_, "Vote does not have ancestry with target block: " @@ -1093,6 +1098,7 @@ namespace kagome::consensus::grandpa { // Check if voter is contained in current voter set auto index_and_weight_opt = voter_set_->indexAndWeight(vote.id); if (!index_and_weight_opt) { + SL_DEBUG(logger_, "Voter {} is not in the current voter set", vote.id.toHex()); return VotingRoundError::UNKNOWN_VOTER; } const auto &[index, weight] = index_and_weight_opt.value(); diff --git a/core/consensus/grandpa/voting_round_error.cpp b/core/consensus/grandpa/voting_round_error.cpp index 45d1e70c4d..e065e740f5 100644 --- a/core/consensus/grandpa/voting_round_error.cpp +++ b/core/consensus/grandpa/voting_round_error.cpp @@ -32,7 +32,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::grandpa, VotingRoundError, e) { return "New round has an abnormal voter set id, expected " "equal or greater by one than the previous voter set id"; case E::UNKNOWN_VOTER: - return "Provided vote is the vote of unknown voter"; + return "Provided vote's voter is unknown"; case E::ZERO_WEIGHT_VOTER: return "Provided vote of disabled (zero weight) voter"; case E::DUPLICATED_VOTE: diff --git a/core/consensus/validation/babe_block_validator.cpp b/core/consensus/validation/babe_block_validator.cpp index 2e547e8ae9..e79dd2283e 100644 --- a/core/consensus/validation/babe_block_validator.cpp +++ b/core/consensus/validation/babe_block_validator.cpp @@ -70,14 +70,16 @@ namespace kagome::consensus { // @see // https://github.com/paritytech/substrate/blob/polkadot-v0.9.8/client/consensus/babe/src/verification.rs#L111 - if (babe_header.needAuthorCheck()) { - if ((not babe_header.needVRFCheck() - and configuration_->allowed_slots - != primitives::AllowedSlots::PrimaryAndSecondaryPlainSlots) - or (babe_header.needVRFCheck() - and configuration_->allowed_slots - != primitives::AllowedSlots:: - PrimaryAndSecondaryVRFSlots)) { + if (babe_header.isProducedInSecondarySlot()) { + bool plainAndAllowed = + babe_header.slotType() == SlotType::SecondaryPlain + && configuration_->allowed_slots + == primitives::AllowedSlots::PrimaryAndSecondaryPlainSlots; + bool vrfAndAllowed = + babe_header.slotType() == SlotType::SecondaryVRF + && configuration_->allowed_slots + == primitives::AllowedSlots::PrimaryAndSecondaryVRFSlots; + if (!plainAndAllowed and !vrfAndAllowed) { SL_WARN(log_, "Secondary slots assignments disabled"); return ValidationError::SECONDARY_SLOT_ASSIGNMENTS_DISABLED; } diff --git a/core/network/impl/stream_engine.hpp b/core/network/impl/stream_engine.hpp index 2301d7f4a1..22a1dedde4 100644 --- a/core/network/impl/stream_engine.hpp +++ b/core/network/impl/stream_engine.hpp @@ -485,7 +485,7 @@ namespace kagome::network { } if (!stream_res) { - self->logger_->error( + self->logger_->debug( "Could not send message to new {} stream with {}: {}", protocol->protocol(), peer_id, From 465fb12745a259ffaa563a08d5471c87005939f4 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 6 Oct 2022 16:51:15 +0300 Subject: [PATCH 64/74] Cleaning up --- .../authority/impl/authority_manager_impl.cpp | 530 ++++-------------- .../authority/impl/authority_manager_impl.hpp | 4 - 2 files changed, 114 insertions(+), 420 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index d8983b08f4..195d47dce4 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -325,55 +325,13 @@ namespace kagome::authority { bool AuthorityManagerImpl::prepare() { const auto finalized_block = block_tree_->getLastFinalized(); - - auto res = persistent_storage_->tryLoad(storage_key); + auto res = initializeAt(finalized_block); if (!res) { SL_ERROR(log_, - "Error loading authority manager root from DB: {}", + "Error initializing authority manager: {}", res.error().message()); - return false; - } - auto &root_opt = res.value(); - if (root_opt) { - auto [current_set, scheduled_changes] = - scale::decode>>( - root_opt.value()) - .value(); - scheduled_changes_ = std::move(scheduled_changes); - current_set_ = std::move(current_set); - current_block_ = scheduled_changes_->block; - } else if (finalized_block.number == 0) { - primitives::BlockInfo genesis_block_info{ - block_tree_->getGenesisBlockHash(), 0}; - scheduled_changes_ = std::make_unique( - ScheduleTree{.payload{ScheduledChangeEntry{ - .new_authorities{primitives::AuthoritySet( - 0, grandpa_api_->authorities(0).value())}, - .scheduled_at{primitives::BlockInfo{ - 0, block_tree_->getGenesisBlockHash()}}, - .activates_at = 0}}, - .block = genesis_block_info, - .children = {}, - .parent = nullptr}); - } else { - SL_ERROR( - log_, - "Failed to initialize authority manager; Try recovering the state."); - return false; } - SL_DEBUG(log_, "Initialized authority manager at block {} with set id {}", - current_block_, - current_set_.id); - return true; - // - // auto res = initializeAt(finalized_block); - // if (!res) { - // SL_ERROR(log_, - // "Error initializing authority manager: {}", - // res.error().message()); - // } - // return res.has_value(); + return res.has_value(); } outcome::result AuthorityManagerImpl::initializeAt( @@ -514,32 +472,24 @@ namespace kagome::authority { OUTCOME_TRY(initial_authorities, grandpa_api_->authorities(genesis_hash)); primitives::BlockInfo genesis_info{0, block_tree_->getGenesisBlockHash()}; - scheduled_changes_ = std::make_unique(ScheduleTree{ - .payload{ScheduledChangeEntry{ - .new_authorities{primitives::AuthoritySet(0, initial_authorities)}, - .scheduled_at{genesis_info}, - .activates_at = 0}}, - .block = genesis_info, - .children = {}, - .parent = nullptr}); + + root_ = ScheduleNode::createAsRoot( + std::make_shared(0, initial_authorities), + {0, genesis_hash}); SL_INFO(log_, "Recovering authority manager state... (might take a few minutes)"); // if state is pruned if (header_repo_->getBlockHeader(1).has_error()) { SL_WARN(log_, - "Can't recalculate authority set id on a prune state, fall back " - "to fetching from runtime"); + "Can't recalculate authority set id on a prune state, fall" + " back to fetching from runtime"); return clearScheduleGraphRoot(*persistent_storage_); } auto start = std::chrono::steady_clock::now(); for (primitives::BlockNumber number = 0; number <= last_finalized_number; number++) { - auto header_res = header_repo_->getBlockHeader(number); - if (!header_res) - continue; // Temporary workaround about the justification pruning bug - auto &header = header_res.value(); - + OUTCOME_TRY(header, header_repo_->getBlockHeader(number)); OUTCOME_TRY(hash, header_repo_->getHashByNumber(number)); primitives::BlockInfo info{number, hash}; @@ -564,59 +514,7 @@ namespace kagome::authority { start = end; } } - return outcome::success(); - // - // root_ = ScheduleNode::createAsRoot( - // std::make_shared(0, - // initial_authorities), {0, genesis_hash}); - // SL_INFO(log_, - // "Recovering authority manager state... (might take a few - // minutes)"); - // // if state is pruned - // if (header_repo_->getBlockHeader(1).has_error()) { - // SL_WARN(log_, - // "Can't recalculate authority set id on a prune state, fall - // back " "to fetching from runtime"); - // return clearScheduleGraphRoot(*persistent_storage_); - // } - // - // auto start = std::chrono::steady_clock::now(); - // for (primitives::BlockNumber number = 0; number <= - // last_finalized_number; - // number++) { - // auto header_res = header_repo_->getBlockHeader(number); - // if (!header_res) - // continue; // Temporary workaround about the justification pruning - // bug - // auto &header = header_res.value(); - // - // OUTCOME_TRY(hash, header_repo_->getHashByNumber(number)); - // primitives::BlockInfo info{number, hash}; - // - // for (auto &msg : header.digest) { - // if (auto consensus_msg = boost::get(&msg); - // consensus_msg != nullptr) { - // onConsensus(info, *consensus_msg).value(); - // } - // } - // auto justification_res = block_tree_->getBlockJustification(hash); - // if (justification_res.has_value()) prune(info); - // auto end = std::chrono::steady_clock::now(); - // auto duration = end - start; - // using namespace std::chrono_literals; - // // 5 seconds is nothing special, just a random more-or-like - // convenient - // // duration. - // if (duration > 5s) { - // SL_VERBOSE(log_, - // "Processed {} out of {} blocks", - // number, - // last_finalized_number); - // start = end; - // } - // } - // return outcome::success(); } primitives::BlockInfo AuthorityManagerImpl::base() const { @@ -630,226 +528,102 @@ namespace kagome::authority { std::optional> AuthorityManagerImpl::authorities(const primitives::BlockInfo &target_block, IsBlockFinalized finalized) const { - auto ancestor = findClosestAncestor(*scheduled_changes_, target_block); - assert(ancestor->block.number <= target_block.number); - if (ancestor == nullptr) return std::nullopt; - if (ancestor->payload.activates_at <= target_block.number) { - return std::make_shared( - ancestor->payload.new_authorities); + auto node = getAppropriateAncestor(target_block); + + if (node == nullptr) { + return std::nullopt; } - if (ancestor->parent == scheduled_changes_.get()) { - return std::make_shared(current_set_); + + auto adjusted_node = node->makeDescendant(target_block, true); + + if (adjusted_node->enabled) { + // Original authorities + SL_DEBUG(log_, + "Pick authority set with id {} for block {}", + adjusted_node->current_authorities->id, + target_block); + for (auto &authority : adjusted_node->current_authorities->authorities) { + SL_TRACE(log_, "Authority {}: {}", authority.id.id, authority.weight); + } + return adjusted_node->current_authorities; } - BOOST_ASSERT(ancestor->parent != nullptr); - return std::make_shared( - ancestor->parent->payload.new_authorities); - // - // auto node = getAppropriateAncestor(target_block); - // - // if (node == nullptr) { - // return std::nullopt; - // } - // - // IsBlockFinalized node_in_finalized_chain = - // node->current_block == target_block - // ? (bool)finalized - // : node->current_block.number - // <= block_tree_->getLastFinalized().number; - // - // auto adjusted_node = - // node->makeDescendant(target_block, node_in_finalized_chain); - // - // if (adjusted_node->enabled) { - // // Original authorities - // SL_DEBUG(log_, - // "Pick authority set with id {} for block {}", - // adjusted_node->current_authorities->id, - // target_block); - // // Since getAppropriateAncestor worked normally on this block - // auto header = - // block_tree_->getBlockHeader(target_block.hash).value(); if (auto - // id_from_storage_res = readSetIdFromRuntime(header); - // id_from_storage_res) { - // auto &id_from_storage_opt = id_from_storage_res.value(); - // if (id_from_storage_opt) { - // SL_DEBUG(log_, - // "Pick authority set id from runtime: {}", - // id_from_storage_opt.value()); - // } else { - // SL_DEBUG(log_, - // "Failed to pick authority set id from runtime: not - // found in " "trie storage"); - // } - // } else { - // SL_DEBUG(log_, - // "Failed to obtain authority set id from runtime: ", - // id_from_storage_res.error().message()); - // } - // for (auto &authority : - // adjusted_node->current_authorities->authorities) { - // SL_TRACE(log_, "Authority {}: {}", authority.id.id, - // authority.weight); - // } - // return adjusted_node->current_authorities; - // } - // - // // Zero-weighted authorities - // auto authorities = std::make_shared( - // *adjusted_node->current_authorities); - // std::for_each(authorities->begin(), - // authorities->end(), - // [](auto &authority) { authority.weight = 0; }); - // return authorities; + + // Zero-weighted authorities + auto authorities = std::make_shared( + *adjusted_node->current_authorities); + std::for_each(authorities->begin(), + authorities->end(), + [](auto &authority) { authority.weight = 0; }); + return authorities; } outcome::result AuthorityManagerImpl::applyScheduledChange( const primitives::BlockInfo &block, const primitives::AuthorityList &authorities, primitives::BlockNumber activate_at) { - if (activate_at < current_block_.number) { - SL_ERROR(log_, - "Error: scheduling changes for a block #{} earlier than the " - "current root #{}", - activate_at, - current_block_.number); + KAGOME_PROFILE_START(get_appropriate_ancestor) + auto ancestor_node = getAppropriateAncestor(block); + KAGOME_PROFILE_END(get_appropriate_ancestor) + + if (not ancestor_node) { return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; } - ForkTree *nearest_change_node = nullptr; - scheduled_changes_->forEachLeaf( - [this, &block, &nearest_change_node](auto &leaf) { - if (leaf.block.number <= block.number - && block_tree_->hasDirectChain(leaf.block.hash, block.hash)) { - nearest_change_node = &leaf; - return true; - } - return false; - }); + SL_DEBUG(log_, + "Authorities for block {} found on block {} with set id {}", + block, + ancestor_node->current_block, + ancestor_node->current_authorities->id); - if (nearest_change_node == nullptr) { - SL_ERROR(log_, - "Neither of pending scheduled changes could be the ancestor of " - "the proposed one (scheduled at #{}).", - block.number); - return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; - } + auto schedule_change = [&](const std::shared_ptr &node) + -> outcome::result { + auto new_authorities = std::make_shared( + node->current_authorities->id + 1, authorities); - auto &nearest_change = nearest_change_node->payload; - // the earliest change encountered in a block is respected - if (nearest_change.scheduled_at.number == block.number) { - SL_DEBUG(log_, - "Ignore scheduled change at {} as this block already " - "schedules a change", - block); - } else if (nearest_change_node->payload.activates_at >= block.number) { - SL_DEBUG( + node->action = + ScheduleNode::ScheduledChange{activate_at, new_authorities}; + + SL_VERBOSE( log_, - "Ignore scheduled change at {} as the previous scheduled change " - "is not applied yet", - block); + "Authority set change is scheduled after block #{} (set id={})", + activate_at, + new_authorities->id); + + size_t index = 0; + for (auto &authority : *new_authorities) { + SL_DEBUG(log_, + "New authority ({}/{}): id={} weight={}", + ++index, + new_authorities->authorities.size(), + authority.id.id, + authority.weight); + } + + return outcome::success(); + }; + + if (ancestor_node->current_block == block) { + OUTCOME_TRY(schedule_change(ancestor_node)); } else { + KAGOME_PROFILE_START(make_descendant) + auto new_node = ancestor_node->makeDescendant(block, true); + KAGOME_PROFILE_END(make_descendant) SL_DEBUG(log_, - "Applying scheduled change with id {} on block {} to activate " - "at block {}", - nearest_change.new_authorities.id + 1, + "Make a schedule node for block {}, with actual set id {}", block, - activate_at); - size_t i = 0; - SL_TRACE(log_, "Authorities:"); - for(auto& authority: authorities) { - SL_TRACE(log_, "{} - {}", i++, authority.id.id.toHex()); - } + new_node->current_authorities->id); + + KAGOME_PROFILE_START(schedule_change) + OUTCOME_TRY(schedule_change(new_node)); + KAGOME_PROFILE_END(schedule_change) - nearest_change_node->children.emplace_back( - std::unique_ptr>{ - new ForkTree{ - .payload = - ScheduledChangeEntry{ - .new_authorities = - primitives::AuthoritySet{ - .id = nearest_change.new_authorities.id + 1, - .authorities = authorities, - }, - .scheduled_at = block, - .activates_at = activate_at}, - .block = block, - .children{}, - .parent = nearest_change_node}}); + // Reorganize ancestry + KAGOME_PROFILE_START(reorganize) + reorganize(ancestor_node, new_node); + KAGOME_PROFILE_END(reorganize) } + return outcome::success(); - // - // KAGOME_PROFILE_START(get_appropriate_ancestor) - // auto ancestor_node = getAppropriateAncestor(block); - // KAGOME_PROFILE_END(get_appropriate_ancestor) - // - // if (not ancestor_node) { - // return AuthorityManagerError::ORPHAN_BLOCK_OR_ALREADY_FINALIZED; - // } - // - // SL_DEBUG(log_, - // "Authorities for block {} found on block {} with set id {}", - // block, - // ancestor_node->current_block, - // ancestor_node->current_authorities->id); - // - // auto schedule_change = [&](const std::shared_ptr &node) - // -> outcome::result { - // auto new_authorities = std::make_shared( - // node->current_authorities->id + 1, authorities); - // - // node->action = - // ScheduleNode::ScheduledChange{activate_at, new_authorities}; - // - // SL_VERBOSE( - // log_, - // "Authority set change is scheduled after block #{} (set id={})", - // activate_at, - // new_authorities->id); - // - // size_t index = 0; - // for (auto &authority : *new_authorities) { - // SL_DEBUG(log_, - // "New authority ({}/{}): id={} weight={}", - // ++index, - // new_authorities->authorities.size(), - // authority.id.id, - // authority.weight); - // } - // - // return outcome::success(); - // }; - // - // /// TODO(Harrm): Should account for finality when fetching an - // authority set - // /// for some purposes, but not when scheduling further changes - // IsBlockFinalized is_ancestor_node_finalized = true; - // - // if (ancestor_node->current_block == block) { - // ancestor_node->adjust(is_ancestor_node_finalized); - // - // OUTCOME_TRY(schedule_change(ancestor_node)); - // } else { - // KAGOME_PROFILE_START(make_descendant) - // auto new_node = - // ancestor_node->makeDescendant(block, - // is_ancestor_node_finalized); - // KAGOME_PROFILE_END(make_descendant) - // SL_DEBUG(log_, - // "Make a schedule node for block {}, with actual set id {}", - // block, - // new_node->current_authorities->id); - // - // KAGOME_PROFILE_START(schedule_change) - // OUTCOME_TRY(schedule_change(new_node)); - // KAGOME_PROFILE_END(schedule_change) - // - // // Reorganize ancestry - // KAGOME_PROFILE_START(reorganize) - // reorganize(ancestor_node, new_node); - // KAGOME_PROFILE_END(reorganize) - // } - // - // return outcome::success(); } outcome::result AuthorityManagerImpl::applyForcedChange( @@ -1079,20 +853,15 @@ namespace kagome::authority { }, [this, &block](const primitives::OnDisabled &msg) { SL_DEBUG(log_, "OnDisabled {}", msg.authority_index); - // return applyOnDisabled(block, msg.authority_index); - return outcome::success(); + return applyOnDisabled(block, msg.authority_index); }, [this, &block](const primitives::Pause &msg) { SL_DEBUG(log_, "Pause {}", msg.subchain_length); - // return applyPause(block, block.number + - // msg.subchain_length); - return outcome::success(); + return applyPause(block, block.number + msg.subchain_length); }, [this, &block](const primitives::Resume &msg) { SL_DEBUG(log_, "Resume {}", msg.subchain_length); - // return applyResume(block, block.number + - // msg.subchain_length); - return outcome::success(); + return applyResume(block, block.number + msg.subchain_length); }, [](auto &) { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; @@ -1113,49 +882,6 @@ namespace kagome::authority { } void AuthorityManagerImpl::prune(const primitives::BlockInfo &block) { - if (block.number <= scheduled_changes_->block.number) { - return; - } - auto ancestor = findClosestAncestor(*scheduled_changes_, block); - if (ancestor == scheduled_changes_.get()) { - if (scheduled_changes_->block.number - < scheduled_changes_->payload.activates_at - && block.number >= scheduled_changes_->payload.activates_at) - current_set_ = scheduled_changes_->payload.new_authorities; - scheduled_changes_->block = block; - current_block_ = block; - } - if (ancestor->block < block) { - ancestor->block = block; - } - - if (ancestor->payload.activates_at > block.number) { - if (ancestor->parent != nullptr) { - current_set_ = ancestor->parent->payload.new_authorities; - } else { - assert(ancestor == scheduled_changes_.get()); - // current set remains unchanged - } - } else { - current_set_ = ancestor->payload.new_authorities; - } - - std::unique_ptr new_root{}; - if (ancestor->parent != nullptr) { - new_root = ancestor->parent->detachChild(*ancestor); - } else { - assert(ancestor == scheduled_changes_.get()); - new_root = std::move(scheduled_changes_); - } - assert(new_root->parent == nullptr); - current_block_ = new_root->block; - scheduled_changes_ = std::move(new_root); - persistent_storage_ - ->put(storage_key, - Buffer{scale::encode(current_set_, scheduled_changes_).value()}) - .value(); - return; - if (block == root_->current_block) { return; } @@ -1270,69 +996,41 @@ namespace kagome::authority { } void AuthorityManagerImpl::cancel(const primitives::BlockInfo &block) { - auto ancestor = findClosestAncestor(*scheduled_changes_, block); + auto ancestor = getAppropriateAncestor(block); + if (ancestor == nullptr) { - SL_TRACE(log_, - "Can't cancel scheduled changes for {}, block is either " - "finalized or unknown", - block); + SL_TRACE(log_, "No scheduled changes on block {}: no ancestor", block); return; } - assert(ancestor->block.number <= block.number); - if (ancestor == scheduled_changes_.get()) { + + if (ancestor == root_) { + // Can't remove root SL_TRACE(log_, - "Can't cancel scheduled changes for {}, block is the finalized " - "root of schedule changes tree", + "Can't cancel scheduled changes on block {}: it is root", block); return; } - assert(ancestor->parent != nullptr); - if (ancestor->block == block) { - ancestor->parent->detachChild(*ancestor); - return; + + if (ancestor->current_block == block) { + ancestor = std::const_pointer_cast(ancestor->parent.lock()); } - for (auto &child : ancestor->children) { - if (block_tree_->hasDirectChain(block.hash, child->block.hash)) { - ancestor->detachChild(*child); + + auto it = std::find_if(ancestor->descendants.begin(), + ancestor->descendants.end(), + [&block](std::shared_ptr node) { + return node->current_block == block; + }); + + if (it != ancestor->descendants.end()) { + if (not(*it)->descendants.empty()) { + // Has descendants - is not a leaf + SL_TRACE(log_, "No scheduled changes on block {}: not found", block); + return; } - } - // auto ancestor = getAppropriateAncestor(block); - // - // if (ancestor == nullptr) { - // SL_TRACE(log_, "No scheduled changes on block {}: no ancestor", - // block); return; - // } - // - // if (ancestor == root_) { - // // Can't remove root - // SL_TRACE(log_, - // "Can't cancel scheduled changes on block {}: it is root", - // block); - // return; - // } - // - // if (ancestor->current_block == block) { - // ancestor = - // std::const_pointer_cast(ancestor->parent.lock()); - // } - // - // auto it = std::find_if(ancestor->descendants.begin(), - // ancestor->descendants.end(), - // [&block](std::shared_ptr node) { - // return node->current_block == block; - // }); - // - // if (it != ancestor->descendants.end()) { - // if (not(*it)->descendants.empty()) { - // // Has descendants - is not a leaf - // SL_TRACE(log_, "No scheduled changes on block {}: not found", - // block); return; - // } - // - // SL_DEBUG(log_, "Scheduled changes on block {} has removed", block); - // ancestor->descendants.erase(it); - // } + SL_DEBUG(log_, "Scheduled changes on block {} has removed", block); + ancestor->descendants.erase(it); + } } ScheduleTree *AuthorityManagerImpl::findClosestAncestor( diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 36d7269974..992a22e261 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -146,10 +146,6 @@ namespace kagome::authority { ScheduleTree const ¤t, primitives::BlockInfo const &block) const; - std::unique_ptr scheduled_changes_; - primitives::AuthoritySet current_set_; - primitives::BlockInfo current_block_; - Config config_; std::shared_ptr block_tree_; std::shared_ptr trie_storage_; From 9355c209a9018a159c6d5fbadf8c254507994ab3 Mon Sep 17 00:00:00 2001 From: Harrm Date: Wed, 12 Oct 2022 17:32:36 +0300 Subject: [PATCH 65/74] Fix CI --- .../consensus/authority/authority_manager.hpp | 8 ----- .../authority/impl/authority_manager_impl.cpp | 1 - core/utils/CMakeLists.txt | 1 + core/utils/storage_explorer.cpp | 29 ++----------------- .../authority/authority_manager_test.cpp | 5 +++- 5 files changed, 7 insertions(+), 37 deletions(-) diff --git a/core/consensus/authority/authority_manager.hpp b/core/consensus/authority/authority_manager.hpp index abf1f4bd9e..a07d69d49d 100644 --- a/core/consensus/authority/authority_manager.hpp +++ b/core/consensus/authority/authority_manager.hpp @@ -25,14 +25,6 @@ namespace kagome::authority { using IsBlockFinalized = Tagged; - /** - * Obtain the current authority set id from the runtime storage - */ - outcome::result> - fetchSetIdFromTrieStorage(storage::trie::TrieStorage const &trie_storage, - crypto::Hasher const &hasher, - storage::trie::RootHash const &state); - class AuthorityManager { public: virtual ~AuthorityManager() = default; diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 95da688bd6..b6762168ec 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -858,7 +858,6 @@ namespace kagome::authority { block, msg.authorities, block.number + msg.subchain_length); }, [this, &block](const primitives::ForcedChange &msg) { - throw std::runtime_error{"not implemented"}; return applyForcedChange( block, msg.authorities, msg.delay_start, msg.subchain_length); }, diff --git a/core/utils/CMakeLists.txt b/core/utils/CMakeLists.txt index e593f1e1e7..3bcc7b1788 100644 --- a/core/utils/CMakeLists.txt +++ b/core/utils/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(storage_explorer ) target_link_libraries(storage_explorer application_injector + authority_manager ) add_backward(storage_explorer) diff --git a/core/utils/storage_explorer.cpp b/core/utils/storage_explorer.cpp index 0ee998f1c3..b3971efb7e 100644 --- a/core/utils/storage_explorer.cpp +++ b/core/utils/storage_explorer.cpp @@ -414,37 +414,13 @@ class SearchChainCommand : public Command { consensus_digest->decode()); if (decoded.consensus_engine_id == kagome::primitives::kGrandpaEngineId) { - auto info = reportAuthorityUpdate( - out, header.number, decoded.asGrandpaDigest()); - if (info.id_changed) { - auto batch_res = - trie_storage->getEphemeralBatchAt(header.state_root); - if (!batch_res) { - std::cerr << "Error fetching trie batch for state " - << header.state_root.toHex() << ": " - << batch_res.error().message() << "\n"; - continue; - } - auto res = kagome::authority::fetchSetIdFromTrieStorage( - *batch_res.value(), *hasher, header.state_root); - if (!res) { - std::cerr << "Error fetching authority set id from storage: " - << res.error().message() << "\n"; - continue; - } - std::cout << "Set id fetched from storage: " - << res.value().value_or(-1) << "\n"; - } + reportAuthorityUpdate(out, header.number, decoded.asGrandpaDigest()); } } } } - struct AuthorityUpdateInfo { - bool id_changed = false; - }; - - AuthorityUpdateInfo reportAuthorityUpdate(std::ostream &out, + void reportAuthorityUpdate(std::ostream &out, BlockNumber digest_origin, GrandpaDigest const &digest) const { using namespace kagome::primitives; @@ -481,7 +457,6 @@ class SearchChainCommand : public Command { out << "Disabled at " << digest_origin << " for authority " << disabled->authority_index << "\n"; } - return {.id_changed = has_id_change}; } std::shared_ptr block_storage; diff --git a/test/core/consensus/authority/authority_manager_test.cpp b/test/core/consensus/authority/authority_manager_test.cpp index 8a952bd9d8..c45df6eccd 100644 --- a/test/core/consensus/authority/authority_manager_test.cpp +++ b/test/core/consensus/authority/authority_manager_test.cpp @@ -21,7 +21,7 @@ #include "primitives/digest.hpp" #include "storage/in_memory/in_memory_storage.hpp" #include "storage/predefined_keys.hpp" -#include "runtime/executor.hpp" +#include "runtime/common/executor.hpp" #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/outcome/dummy_error.hpp" @@ -80,6 +80,9 @@ class AuthorityManagerTest : public testing::Test { EXPECT_CALL(*grandpa_api, authorities(_)) .WillRepeatedly(Return(authorities->authorities)); + EXPECT_CALL(*grandpa_api, current_set_id(_)) + .WillRepeatedly(Return(42)); + hasher = std::make_shared(); EXPECT_CALL(*hasher, twox_128(_)).WillRepeatedly(Return(common::Hash128{})); From 74bceb3c85ea5f6c081d1f41da46a626bc59a2c5 Mon Sep 17 00:00:00 2001 From: Harrm Date: Wed, 12 Oct 2022 18:19:33 +0300 Subject: [PATCH 66/74] Merge with master --- .../authority/impl/authority_manager_impl.cpp | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 0b0f93d3f0..9bf67b57b4 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -15,6 +15,7 @@ #include "blockchain/block_header_repository.hpp" #include "blockchain/block_tree.hpp" #include "blockchain/block_tree_error.hpp" +#include "common/monadic_utils.hpp" #include "common/visitor.hpp" #include "consensus/authority/authority_manager_error.hpp" #include "consensus/authority/authority_update_observer_error.hpp" @@ -347,6 +348,9 @@ namespace kagome::authority { block_tree_->getBlockHeader(graph_root_block.hash)); auto set_id_from_runtime_res = readSetIdFromRuntime(root_header); + auto set_id_from_runtime_opt = set_id_from_runtime_res.has_value() + ? set_id_from_runtime_res.value() + : std::nullopt; OUTCOME_TRY(opt_root, fetchScheduleGraphRoot(*persistent_storage_)); auto last_finalized_block = block_tree_->getLastFinalized(); @@ -357,14 +361,14 @@ namespace kagome::authority { // TODO(Harrm): #1334 // Correction to bypass the bug where after finishing syncing // and restarting the node we get a set id off by one - if (set_id_from_runtime_res.has_value() + if (set_id_from_runtime_opt.has_value() && opt_root.value()->current_authorities->id - == set_id_from_runtime_res.value() - 1) { + == set_id_from_runtime_opt.value() - 1) { auto &authority_list = opt_root.value()->current_authorities->authorities; opt_root.value()->current_authorities = std::make_shared( - set_id_from_runtime_res.value(), authority_list); + set_id_from_runtime_opt.value(), authority_list); } root_ = std::move(opt_root.value()); @@ -379,7 +383,8 @@ namespace kagome::authority { std::make_shared( 0, std::move(initial_authorities)), {0, genesis_hash}); - } else if (set_id_from_runtime_res.has_value()){ + } else if (set_id_from_runtime_res.has_value() + && set_id_from_runtime_opt.has_value()) { SL_WARN( log_, "Storage does not contain valid info about the root authority set; " @@ -389,7 +394,7 @@ namespace kagome::authority { grandpa_api_->authorities(graph_root_block.hash)); auto authority_set = std::make_shared( - set_id_from_runtime_res.value(), std::move(authorities)); + set_id_from_runtime_opt.value(), std::move(authorities)); root_ = authority::ScheduleNode::createAsRoot(authority_set, graph_root_block); @@ -399,7 +404,9 @@ namespace kagome::authority { "storage", root_->current_authorities->id); } else { - SL_ERROR(log_, "Failed to initialize authority manager; Try running recovery mode"); + SL_ERROR( + log_, + "Failed to initialize authority manager; Try running recovery mode"); return set_id_from_runtime_res.as_failure(); } @@ -743,8 +750,8 @@ namespace kagome::authority { return outcome::success(); }; - auto new_node = ancestor_node->makeDescendant( - {delay_start, delay_start_hash}, true); + auto new_node = + ancestor_node->makeDescendant({delay_start, delay_start_hash}, true); OUTCOME_TRY(force_change(new_node)); @@ -921,8 +928,7 @@ namespace kagome::authority { }); } else if (message.consensus_engine_id == primitives::kBabeEngineId || message.consensus_engine_id - == primitives::kUnsupportedEngineId_BEEF - ) { + == primitives::kUnsupportedEngineId_BEEF) { // ignore return outcome::success(); From 8dd155d0930d27a676e65097832afef3a72cbf06 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 13 Oct 2022 15:07:45 +0300 Subject: [PATCH 67/74] Fix merge --- housekeeping/ci_install_mold.sh | 11 +++++++++++ housekeeping/indocker.sh | 5 ++--- .../cmake/clang-11-cpp17-toolchain.cmake | 1 - .../cmake/clang-14-cpp17-toolchain.cmake | 1 - .../external-project-test/cmake/cpp17-toolchain.cmake | 4 ---- 5 files changed, 13 insertions(+), 9 deletions(-) create mode 100755 housekeeping/ci_install_mold.sh delete mode 100644 test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake delete mode 100644 test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake delete mode 100644 test/external-project-test/cmake/cpp17-toolchain.cmake diff --git a/housekeeping/ci_install_mold.sh b/housekeeping/ci_install_mold.sh new file mode 100755 index 0000000000..c7f7d0c567 --- /dev/null +++ b/housekeeping/ci_install_mold.sh @@ -0,0 +1,11 @@ +#!/bin/bash -xe + +if [ "$CI" != "true" ] || [ "$GITHUB_ACTIONS" != "true" ]; then + echo "CI=true GITHUB_ACTIONS=true are required" + exit -1 +fi + +wget -q -O- https://github.com/rui314/mold/releases/download/v1.3.1/mold-1.3.1-x86_64-linux.tar.gz | tar -C /usr/local --strip-components=1 -xzf - +if [ "$1" == "--make-default" ]; then + ln -sf /usr/local/bin/mold $(realpath /usr/bin/ld) +fi diff --git a/housekeeping/indocker.sh b/housekeeping/indocker.sh index 0131f63e72..7a7a89bc80 100755 --- a/housekeeping/indocker.sh +++ b/housekeeping/indocker.sh @@ -18,7 +18,7 @@ docker build -t soramitsu/kagome:local-dev $DIR rm $DIR/Dockerfile -docker run -it --rm \ +docker run -i --rm \ --cap-add SYS_PTRACE \ -v /tmp/cache/hunter:/root/.hunter \ -w /workdir \ @@ -26,5 +26,4 @@ docker run -it --rm \ -e CODECOV_TOKEN \ -e SONAR_TOKEN \ $CI_ENV \ - soramitsu/kagome:local-dev \ - /bin/bash + soramitsu/kagome:local-dev diff --git a/test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake b/test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake deleted file mode 100644 index 536e4a008d..0000000000 --- a/test/external-project-test/cmake/clang-11-cpp17-toolchain.cmake +++ /dev/null @@ -1 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/cpp17-toolchain.cmake") \ No newline at end of file diff --git a/test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake b/test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake deleted file mode 100644 index 536e4a008d..0000000000 --- a/test/external-project-test/cmake/clang-14-cpp17-toolchain.cmake +++ /dev/null @@ -1 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/cpp17-toolchain.cmake") \ No newline at end of file diff --git a/test/external-project-test/cmake/cpp17-toolchain.cmake b/test/external-project-test/cmake/cpp17-toolchain.cmake deleted file mode 100644 index 5c08ad5d6c..0000000000 --- a/test/external-project-test/cmake/cpp17-toolchain.cmake +++ /dev/null @@ -1,4 +0,0 @@ - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) From 25fa77801a20d704e6b75b43c140b698eff52c5a Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 13 Oct 2022 15:19:46 +0300 Subject: [PATCH 68/74] Self review --- .../authority/impl/authority_manager_impl.cpp | 101 ------------------ .../authority/impl/authority_manager_impl.hpp | 14 +-- core/utils/storage_explorer.cpp | 3 - 3 files changed, 1 insertion(+), 117 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 9bf67b57b4..36a4a5c9a6 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -15,7 +15,6 @@ #include "blockchain/block_header_repository.hpp" #include "blockchain/block_tree.hpp" #include "blockchain/block_tree_error.hpp" -#include "common/monadic_utils.hpp" #include "common/visitor.hpp" #include "consensus/authority/authority_manager_error.hpp" #include "consensus/authority/authority_update_observer_error.hpp" @@ -31,81 +30,6 @@ using kagome::primitives::AuthoritySetId; namespace kagome::authority { - struct ScheduledChangeEntry { - primitives::AuthoritySet new_authorities; - primitives::BlockInfo scheduled_at; - primitives::BlockNumber activates_at; - }; - - template - struct ForkTree { - T payload; - primitives::BlockInfo block; - std::vector> children; - ForkTree *parent; - - ForkTree *find(primitives::BlockHash const &block_hash) const { - if (block.hash == block_hash) return this; - for (auto &child : children) { - if (auto *res = child->find(block_hash); res != nullptr) { - return res; - } - } - return nullptr; - } - - bool forEachLeaf(std::function const &f) { - if (children.empty()) { - return f(*this); - } - for (auto &child : children) { - bool res = child->forEachLeaf(f); - if (res) return true; - } - return false; - } - - std::unique_ptr> detachChild(ForkTree &child) { - for (size_t i = 0; i < children.size(); i++) { - auto ¤t_child = children[i]; - if (current_child.get() == &child) { - auto owned_child = std::move(current_child); - owned_child->parent = nullptr; - children.erase(children.begin() + i); - return owned_child; - } - } - return nullptr; - } - }; - - ::scale::ScaleEncoderStream &operator<<(::scale::ScaleEncoderStream &s, - const ScheduledChangeEntry &entry) { - s << entry.scheduled_at << entry.activates_at << entry.new_authorities; - return s; - } - - ::scale::ScaleDecoderStream &operator>>(::scale::ScaleDecoderStream &s, - ScheduledChangeEntry &entry) { - s >> entry.scheduled_at >> entry.activates_at >> entry.new_authorities; - return s; - } - - ::scale::ScaleEncoderStream &operator<<(::scale::ScaleEncoderStream &s, - const ScheduleTree &tree) { - s << tree.block << tree.payload << tree.children; - return s; - } - - ::scale::ScaleDecoderStream &operator>>(::scale::ScaleDecoderStream &s, - ScheduleTree &change) { - s >> change.block >> change.payload >> change.children; - for (auto &child : change.children) { - child->parent = &change; - } - return s; - } - AuthorityManagerImpl::AuthorityManagerImpl( Config config, std::shared_ptr app_state_manager, @@ -1092,29 +1016,4 @@ namespace kagome::authority { ancestor->descendants.erase(it); } } - - ScheduleTree *AuthorityManagerImpl::findClosestAncestor( - ForkTree ¤t, - primitives::BlockInfo const &block) const { - auto *res = findClosestAncestor( - const_cast const &>(current), block); - return const_cast *>(res); - } - - ScheduleTree const *AuthorityManagerImpl::findClosestAncestor( - ForkTree const ¤t, - primitives::BlockInfo const &block) const { - for (auto &child : current.children) { - if (child->block.number < block.number - && block_tree_->hasDirectChain(child->block.hash, block.hash)) { - return findClosestAncestor(*child, block); - } - } - if (current.block.number <= block.number - && block_tree_->hasDirectChain(current.block.hash, block.hash)) { - return ¤t; - } - return nullptr; - } - } // namespace kagome::authority diff --git a/core/consensus/authority/impl/authority_manager_impl.hpp b/core/consensus/authority/impl/authority_manager_impl.hpp index 992a22e261..b3654e4d64 100644 --- a/core/consensus/authority/impl/authority_manager_impl.hpp +++ b/core/consensus/authority/impl/authority_manager_impl.hpp @@ -25,6 +25,7 @@ namespace kagome::blockchain { class BlockTree; class BlockHeaderRepository; } // namespace kagome::blockchain + namespace kagome::primitives { struct BabeConfiguration; } // namespace kagome::primitives @@ -40,10 +41,6 @@ namespace kagome::storage::trie { namespace kagome::authority { - struct ScheduledChangeEntry; - template struct ForkTree; - using ScheduleTree = ForkTree; - class AuthorityManagerImpl : public AuthorityManager, public AuthorityUpdateObserver { public: @@ -137,15 +134,6 @@ namespace kagome::authority { void reorganize(std::shared_ptr node, std::shared_ptr new_node); - - ScheduleTree *findClosestAncestor( - ScheduleTree ¤t, - primitives::BlockInfo const &block) const; - - ScheduleTree const *findClosestAncestor( - ScheduleTree const ¤t, - primitives::BlockInfo const &block) const; - Config config_; std::shared_ptr block_tree_; std::shared_ptr trie_storage_; diff --git a/core/utils/storage_explorer.cpp b/core/utils/storage_explorer.cpp index b3971efb7e..b34c592d69 100644 --- a/core/utils/storage_explorer.cpp +++ b/core/utils/storage_explorer.cpp @@ -424,10 +424,8 @@ class SearchChainCommand : public Command { BlockNumber digest_origin, GrandpaDigest const &digest) const { using namespace kagome::primitives; - bool has_id_change = false; if (auto *scheduled_change = boost::get(&digest); scheduled_change) { - has_id_change = true; out << "ScheduledChange at #" << digest_origin << " for "; if (scheduled_change->subchain_length > 0) { out << "#" << digest_origin + scheduled_change->subchain_length; @@ -438,7 +436,6 @@ class SearchChainCommand : public Command { } else if (auto *forced_change = boost::get(&digest); forced_change) { - has_id_change = true; out << "ForcedChange at " << digest_origin << ", delay starts at #" << forced_change->delay_start << " for " << forced_change->subchain_length << " blocks (so activates at #" From 80502b92c71515a6adf3e1f5854f9ad687bdee02 Mon Sep 17 00:00:00 2001 From: Harrm Date: Fri, 14 Oct 2022 00:06:30 +0300 Subject: [PATCH 69/74] Self review --- core/consensus/babe/types/slot.hpp | 14 +++++++++++++- core/consensus/validation/babe_block_validator.cpp | 8 +++++--- core/primitives/babe_configuration.hpp | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/core/consensus/babe/types/slot.hpp b/core/consensus/babe/types/slot.hpp index 2ee276e941..bdc9383d44 100644 --- a/core/consensus/babe/types/slot.hpp +++ b/core/consensus/babe/types/slot.hpp @@ -19,6 +19,18 @@ namespace kagome::consensus { SecondaryVRF = 3, }; -} + static std::string_view to_string(SlotType s) { + switch (s) { + case SlotType::Primary: + return "Primary"; + case SlotType::SecondaryPlain: + return "Secondary Plain"; + case SlotType::SecondaryVRF: + return "Secondary VRF"; + } + return "Unknown"; + } + +} // namespace kagome::consensus #endif // KAGOME_SLOT_HPP diff --git a/core/consensus/validation/babe_block_validator.cpp b/core/consensus/validation/babe_block_validator.cpp index df58b46d54..0eca129518 100644 --- a/core/consensus/validation/babe_block_validator.cpp +++ b/core/consensus/validation/babe_block_validator.cpp @@ -84,13 +84,15 @@ namespace kagome::consensus { && babe_config.allowed_slots == primitives::AllowedSlots::PrimaryAndSecondaryVRFSlots; if (!plainAndAllowed and !vrfAndAllowed) { + // SL_WARN unwraps to a lambda which cannot capture a local binding, + // thus this copy + auto slot_type = babe_header.slotType(); SL_WARN(log_, "Block {} produced in {} secondary slot, but current " "configuration allows only {}", header.number, - babe_header.slotType(), - babe_config.allowed_slots); - // return ValidationError::SECONDARY_SLOT_ASSIGNMENTS_DISABLED; + to_string(slot_type), + to_string(babe_config.allowed_slots)); } } diff --git a/core/primitives/babe_configuration.hpp b/core/primitives/babe_configuration.hpp index 7ba27eef72..cab2b7c485 100644 --- a/core/primitives/babe_configuration.hpp +++ b/core/primitives/babe_configuration.hpp @@ -6,6 +6,8 @@ #ifndef KAGOME_CORE_PRIMITIVES_BABE_CONFIGURATION_HPP #define KAGOME_CORE_PRIMITIVES_BABE_CONFIGURATION_HPP +#include + #include "common/blob.hpp" #include "consensus/babe/common.hpp" #include "crypto/sr25519_types.hpp" @@ -24,6 +26,18 @@ namespace kagome::primitives { PrimaryAndSecondaryVRFSlots }; + static std::string_view to_string(AllowedSlots s) { + switch(s) { + case AllowedSlots::PrimarySlots: + return "Primary Slots"; + case AllowedSlots::PrimaryAndSecondaryPlainSlots: + return "Primary and Secondary Plain Slots"; + case AllowedSlots::PrimaryAndSecondaryVRFSlots: + return "Primary and Secondary VRF Slots"; + } + return "Unknown"; + } + /// Configuration data used by the BABE consensus engine. struct BabeConfiguration { /// The slot duration in milliseconds for BABE. Currently, only From 979abe13469094e1db0a270bfe459316c1fd3371 Mon Sep 17 00:00:00 2001 From: Harrm Date: Tue, 18 Oct 2022 12:04:22 +0300 Subject: [PATCH 70/74] Enable VoteCryptoProviderImpl::verify --- core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index 7f8b0104cd..6436c7b89e 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -33,7 +33,6 @@ namespace kagome::consensus::grandpa { bool VoteCryptoProviderImpl::verify(const SignedMessage &vote, RoundNumber number) const { - return true; auto payload = scale::encode(vote.message, number, voter_set_->id()).value(); auto verifying_result = From 1a81fb518686d26f079ee82889bb93385a06249f Mon Sep 17 00:00:00 2001 From: Harrm Date: Wed, 19 Oct 2022 14:28:10 +0300 Subject: [PATCH 71/74] Fixes from review --- .gitignore | 4 +- .../authority/impl/authority_manager_impl.cpp | 9 ++- .../authority/impl/schedule_node.hpp | 55 +++---------------- core/consensus/babe/types/slot.hpp | 2 +- core/consensus/grandpa/CMakeLists.txt | 2 +- core/network/impl/stream_engine.hpp | 4 +- core/primitives/babe_configuration.hpp | 2 +- .../common/runtime_environment_factory.cpp | 4 +- .../runtime/common/runtime_instances_pool.hpp | 2 +- 9 files changed, 22 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index 47b24f5ff6..4a6654c9fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ config/__pycahe__ **/__pycache__/* __pycache__/* -build/* -build-*/* +**/build/* +**/build-*/* *.DS_Store *.vscode/ *.swp diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 36a4a5c9a6..dab88c965a 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -243,9 +243,6 @@ namespace kagome::authority { BOOST_UNREACHABLE_RETURN({}) } - static const Buffer storage_key = - Buffer::fromString("test_authority_manager_root"); - AuthorityManagerImpl::~AuthorityManagerImpl() {} bool AuthorityManagerImpl::prepare() { @@ -851,8 +848,10 @@ namespace kagome::authority { return AuthorityUpdateObserverError::UNSUPPORTED_MESSAGE_TYPE; }); } else if (message.consensus_engine_id == primitives::kBabeEngineId - || message.consensus_engine_id - == primitives::kUnsupportedEngineId_BEEF) { + or message.consensus_engine_id + == primitives::kUnsupportedEngineId_BEEF + or message.consensus_engine_id + == primitives::kUnsupportedEngineId_POL1) { // ignore return outcome::success(); diff --git a/core/consensus/authority/impl/schedule_node.hpp b/core/consensus/authority/impl/schedule_node.hpp index 5271568ddf..ad6c7a84e0 100644 --- a/core/consensus/authority/impl/schedule_node.hpp +++ b/core/consensus/authority/impl/schedule_node.hpp @@ -46,69 +46,30 @@ namespace kagome::authority { using NoAction = Empty; struct ScheduledChange { + SCALE_TIE(2); + primitives::BlockNumber applied_block{}; std::shared_ptr new_authorities{}; - - friend inline ::scale::ScaleEncoderStream &operator<<( - ::scale::ScaleEncoderStream &s, const ScheduledChange &change) { - return s << change.applied_block << *change.new_authorities; - } - - friend inline ::scale::ScaleDecoderStream &operator>>( - ::scale::ScaleDecoderStream &s, ScheduledChange &change) { - auto authority_list = std::make_shared(); - s >> change.applied_block >> *authority_list; - change.new_authorities = std::move(authority_list); - return s; - } }; struct ForcedChange { + SCALE_TIE(3); + primitives::BlockNumber delay_start{}; size_t delay_length{}; std::shared_ptr new_authorities{}; - - friend inline ::scale::ScaleEncoderStream &operator<<( - ::scale::ScaleEncoderStream &s, const ForcedChange &change) { - return s << change.delay_start << change.delay_length - << *change.new_authorities; - } - - friend inline ::scale::ScaleDecoderStream &operator>>( - ::scale::ScaleDecoderStream &s, ForcedChange &change) { - auto authority_list = std::make_shared(); - s >> change.delay_start >> change.delay_length >> *authority_list; - change.new_authorities = std::move(authority_list); - return s; - } }; struct Pause { - primitives::BlockNumber applied_block{}; + SCALE_TIE(1); - friend inline ::scale::ScaleEncoderStream &operator<<( - ::scale::ScaleEncoderStream &s, const Pause &change) { - return s << change.applied_block; - } - - friend inline ::scale::ScaleDecoderStream &operator>>( - ::scale::ScaleDecoderStream &s, Pause &change) { - return s >> change.applied_block; - } + primitives::BlockNumber applied_block{}; }; struct Resume { - primitives::BlockNumber applied_block{}; + SCALE_TIE(1); - friend inline ::scale::ScaleEncoderStream &operator<<( - ::scale::ScaleEncoderStream &s, const Resume &change) { - return s << change.applied_block; - } - - friend inline ::scale::ScaleDecoderStream &operator>>( - ::scale::ScaleDecoderStream &s, Resume &change) { - return s >> change.applied_block; - } + primitives::BlockNumber applied_block{}; }; friend inline ::scale::ScaleEncoderStream &operator<<( diff --git a/core/consensus/babe/types/slot.hpp b/core/consensus/babe/types/slot.hpp index bdc9383d44..3a8b4a2ecd 100644 --- a/core/consensus/babe/types/slot.hpp +++ b/core/consensus/babe/types/slot.hpp @@ -19,7 +19,7 @@ namespace kagome::consensus { SecondaryVRF = 3, }; - static std::string_view to_string(SlotType s) { + inline std::string_view to_string(SlotType s) { switch (s) { case SlotType::Primary: return "Primary"; diff --git a/core/consensus/grandpa/CMakeLists.txt b/core/consensus/grandpa/CMakeLists.txt index 2b3c742521..7b8a250315 100644 --- a/core/consensus/grandpa/CMakeLists.txt +++ b/core/consensus/grandpa/CMakeLists.txt @@ -39,7 +39,7 @@ target_link_libraries(voter_set ) add_library(voting_round_error - voting_round_error.cpp + voting_round_error.cpp ) target_link_libraries(voting_round_error outcome diff --git a/core/network/impl/stream_engine.hpp b/core/network/impl/stream_engine.hpp index 1517aef279..b2df558050 100644 --- a/core/network/impl/stream_engine.hpp +++ b/core/network/impl/stream_engine.hpp @@ -521,8 +521,8 @@ namespace kagome::network { } if (!stream_res) { - self->logger_->debug( - "Could not send message to new {} stream with {}: {}", + SL_DEBUG(self->logger_, + "Could not send message to new {} stream with {}: {}", protocol->protocolName(), peer_id, stream_res.error().message()); diff --git a/core/primitives/babe_configuration.hpp b/core/primitives/babe_configuration.hpp index cab2b7c485..3346024066 100644 --- a/core/primitives/babe_configuration.hpp +++ b/core/primitives/babe_configuration.hpp @@ -26,7 +26,7 @@ namespace kagome::primitives { PrimaryAndSecondaryVRFSlots }; - static std::string_view to_string(AllowedSlots s) { + inline std::string_view to_string(AllowedSlots s) { switch(s) { case AllowedSlots::PrimarySlots: return "Primary Slots"; diff --git a/core/runtime/common/runtime_environment_factory.cpp b/core/runtime/common/runtime_environment_factory.cpp index 7330ad1795..5a5dce97c6 100644 --- a/core/runtime/common/runtime_environment_factory.cpp +++ b/core/runtime/common/runtime_environment_factory.cpp @@ -89,7 +89,7 @@ namespace kagome::runtime { if (persistent_) { if (auto res = env.storage_provider->setToPersistentAt(storage_state_); !res) { - parent_factory->logger_->debug( + parent_factory->logger_->error( "Failed to set the storage state to hash {:l} when initializing a " "runtime environment; Reason: {}", storage_state_, @@ -99,7 +99,7 @@ namespace kagome::runtime { } else { if (auto res = env.storage_provider->setToEphemeralAt(storage_state_); !res) { - parent_factory->logger_->debug( + parent_factory->logger_->error( "Failed to set the storage state to hash {:l} when initializing a " "runtime environment; Reason: {}", storage_state_, diff --git a/core/runtime/common/runtime_instances_pool.hpp b/core/runtime/common/runtime_instances_pool.hpp index eb1c80f048..9ddcee686e 100644 --- a/core/runtime/common/runtime_instances_pool.hpp +++ b/core/runtime/common/runtime_instances_pool.hpp @@ -138,4 +138,4 @@ namespace kagome::runtime { } // namespace kagome::runtime -#endif // KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP \ No newline at end of file +#endif // KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP From 78360d5e8784d932bfa200edd8099d4df78f3841 Mon Sep 17 00:00:00 2001 From: Harrm Date: Wed, 26 Oct 2022 12:28:11 +0300 Subject: [PATCH 72/74] Fixes from review --- .../authority/impl/authority_manager_impl.cpp | 30 ++- .../impl/vote_crypto_provider_impl.cpp | 25 +- core/host_api/impl/offchain_extension.cpp | 14 +- .../common/runtime_environment_factory.cpp | 4 +- .../runtime/common/runtime_instances_pool.hpp | 254 +++++++++--------- 5 files changed, 172 insertions(+), 155 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index dab88c965a..038459fa82 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -361,23 +361,25 @@ namespace kagome::authority { AuthorityManagerImpl::readSetIdFromRuntime( primitives::BlockHeader const &header) const { AuthoritySetId set_id{}; + + auto batch_res = trie_storage_->getEphemeralBatchAt(header.state_root); + if (batch_res.has_error()) { + if (batch_res.error() == storage::DatabaseError::NOT_FOUND) { + SL_DEBUG( + log_, + "Failed to fetch set id from trie storage: state {} is not in " + "the storage", + header.state_root); + return std::nullopt; + } + return batch_res.as_failure(); + } + OUTCOME_TRY(hash, primitives::calculateBlockHash(header, *hasher_)); auto set_id_res = grandpa_api_->current_set_id(hash); if (set_id_res) { set_id = set_id_res.value(); } else { - auto batch_res = trie_storage_->getEphemeralBatchAt(header.state_root); - if (batch_res.has_error()) { - if (batch_res.error() == storage::DatabaseError::NOT_FOUND) { - SL_DEBUG( - log_, - "Failed to fetch set id from trie storage: state {} is not in " - "the storage", - header.state_root); - return std::nullopt; - } - return batch_res.as_failure(); - } auto &batch = batch_res.value(); OUTCOME_TRY( @@ -529,7 +531,7 @@ namespace kagome::authority { size_t index = 0; for (auto &authority : *new_authorities) { - SL_DEBUG(log_, + SL_TRACE(log_, "New authority ({}/{}): id={} weight={}", ++index, new_authorities->authorities.size(), @@ -660,7 +662,7 @@ namespace kagome::authority { size_t index = 0; for (auto &authority : *new_authorities) { - SL_DEBUG(log_, + SL_TRACE(log_, "New authority ({}/{}): id={} weight={}", ++index, new_authorities->authorities.size(), diff --git a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp index 6436c7b89e..e5915c0f62 100644 --- a/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp +++ b/core/consensus/grandpa/impl/vote_crypto_provider_impl.cpp @@ -42,18 +42,23 @@ namespace kagome::consensus::grandpa { // calculation errors if (!result) { auto logger = log::createLogger("VoteCryptoProvider", "authority"); - for (auto id = voter_set_->id() - 50; id < voter_set_->id() + 50; id++) { - auto payload = scale::encode(vote.message, number, id).value(); - auto verifying_result = - ed_provider_->verify(vote.signature, payload, vote.id); - if (verifying_result.has_value() and verifying_result.value()) { - SL_DEBUG(logger, - "Correct set id is {}, actual is {}", - id, - voter_set_->id()); - return false; + for (auto n = number - 100; n < number + 100; n++) { + for (auto id = voter_set_->id() - 100; id < voter_set_->id() + 100; id++) { + auto payload = scale::encode(vote.message, n, id).value(); + auto verifying_result = + ed_provider_->verify(vote.signature, payload, vote.id); + if (verifying_result.has_value() and verifying_result.value()) { + SL_DEBUG(logger, + "Correct set id and number are {} {}, actual are {} {}", + id, + n, + voter_set_->id(), + number); + return false; + } } } + SL_DEBUG(logger, "Failed to find correct set id"); } #endif diff --git a/core/host_api/impl/offchain_extension.cpp b/core/host_api/impl/offchain_extension.cpp index 3abec604fa..06c178fe11 100644 --- a/core/host_api/impl/offchain_extension.cpp +++ b/core/host_api/impl/offchain_extension.cpp @@ -178,12 +178,22 @@ namespace kagome::host_api { auto [key_ptr, key_size] = runtime::PtrSize(key); auto key_buffer = memory.loadN(key_ptr, key_size); auto [expected_ptr, expected_size] = runtime::PtrSize(expected); - auto expected_buffer = memory.loadN(expected_ptr, expected_size); + auto expected_encoded = memory.loadN(expected_ptr, expected_size); + auto expected_res = + scale::decode>(expected_encoded); + if (expected_res.has_error()) { + throw std::runtime_error("Invalid encoded data for expected arg"); + } + auto &expected_as_buffer{expected_res.value()}; + std::optional expected_as_view; + if (expected_as_buffer) { + expected_as_view.emplace(expected_as_buffer.value()); + } auto [value_ptr, value_size] = runtime::PtrSize(value); auto value_buffer = memory.loadN(value_ptr, value_size); auto result = worker->localStorageCompareAndSet( - storage_type, key_buffer, expected_buffer, value_buffer); + storage_type, key_buffer, expected_as_view, value_buffer); return result; } diff --git a/core/runtime/common/runtime_environment_factory.cpp b/core/runtime/common/runtime_environment_factory.cpp index 5a5dce97c6..e21a53c524 100644 --- a/core/runtime/common/runtime_environment_factory.cpp +++ b/core/runtime/common/runtime_environment_factory.cpp @@ -89,7 +89,7 @@ namespace kagome::runtime { if (persistent_) { if (auto res = env.storage_provider->setToPersistentAt(storage_state_); !res) { - parent_factory->logger_->error( + SL_ERROR(parent_factory->logger_, "Failed to set the storage state to hash {:l} when initializing a " "runtime environment; Reason: {}", storage_state_, @@ -99,7 +99,7 @@ namespace kagome::runtime { } else { if (auto res = env.storage_provider->setToEphemeralAt(storage_state_); !res) { - parent_factory->logger_->error( + SL_ERROR(parent_factory->logger_, "Failed to set the storage state to hash {:l} when initializing a " "runtime environment; Reason: {}", storage_state_, diff --git a/core/runtime/common/runtime_instances_pool.hpp b/core/runtime/common/runtime_instances_pool.hpp index 9ddcee686e..1ec7fad9af 100644 --- a/core/runtime/common/runtime_instances_pool.hpp +++ b/core/runtime/common/runtime_instances_pool.hpp @@ -1,7 +1,7 @@ /** -* Copyright Soramitsu Co., Ltd. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0 -*/ + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ #ifndef KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP #define KAGOME_CORE_RUNTIME_INSTANCES_POOL_HPP @@ -11,130 +11,130 @@ #include namespace kagome::runtime { - /** - * LRU cache designed for small amounts of data (as its get() is O(N)) - */ - template - struct SmallLruCache final { - public: - static_assert(std::is_unsigned_v); - - struct CacheEntry { - Key key; - Value value; - PriorityType latest_use_tick_; - - bool operator<(const CacheEntry &rhs) const { - return latest_use_tick_ < rhs.latest_use_tick_; - } - }; - - SmallLruCache(size_t max_size) : kMaxSize{max_size} { - BOOST_ASSERT(kMaxSize > 0); - cache_.reserve(kMaxSize); - } - - std::optional> get(const Key &key) { - ticks_++; - if (ticks_ == 0) { - handleTicksOverflow(); - } - for (auto &entry : cache_) { - if (entry.key == key) { - entry.latest_use_tick_ = ticks_; - return entry.value; - } - } - return std::nullopt; - } - - template - void put(const Key &key, ValueArg &&value) { - static_assert(std::is_convertible_v< - ValueArg, - Value> || std::is_constructible_v); - ticks_++; - if (cache_.size() >= kMaxSize) { - auto min = std::min_element(cache_.begin(), cache_.end()); - cache_.erase(min); - } - cache_.push_back(CacheEntry{key, std::forward(value), ticks_}); - } - - private: - void handleTicksOverflow() { - // 'compress' timestamps of entries in the cache (works because we care - // only about their order, not actual timestamps) - std::sort(cache_.begin(), cache_.end()); - for (auto &entry : cache_) { - entry.latest_use_tick_ = ticks_; - ticks_++; - } - } - - const size_t kMaxSize; - // an abstract representation of time to implement 'recency' without - // depending on real time. Incremented on each cache access - PriorityType ticks_{}; - std::vector cache_; - }; - - /** - * @brief Pool of runtime instances - per state. Incapsulates modules cache. - * - */ - class RuntimeInstancesPool final - : public std::enable_shared_from_this { - using ModuleInstancePool = std::stack>; - - public: - using RootHash = storage::trie::RootHash; - using ModuleCache = - SmallLruCache>; - - /** - * @brief Instantiate new or reuse existing ModuleInstance for the provided - * state. - * - * @param state - the merkle trie root of the state containing the code of - * the runtime module we are acquiring an instance of. - * @return pointer to the acquired ModuleInstance if success. Error - * otherwise. - */ - outcome::result> tryAcquire( - const RootHash &state); - /** - * @brief Releases the module instance (returns it to the pool) - * - * @param state - the merkle trie root of the state containing the runtime - * module code we are releasing an instance of. - * @param instance - instance to be released. - */ - void release(const RootHash &state, - std::shared_ptr &&instance); - - /** - * @brief Get the module for state from internal cache - * - * @param state - the state containing the module's code. - * @return Module if any, nullopt otherwise - */ - std::optional> getModule(const RootHash &state); - - /** - * @brief Puts new module into internal cache - * - * @param state - runtime block, by its root hash - * @param module - new module pointer - */ - void putModule(const RootHash &state, std::shared_ptr module); - - private: - std::mutex mt_; - static constexpr size_t MODULES_CACHE_SIZE = 2; - ModuleCache modules_{MODULES_CACHE_SIZE}; - std::map pools_; - }; + /** + * LRU cache designed for small amounts of data (as its get() is O(N)) + */ + template + struct SmallLruCache final { + public: + static_assert(std::is_unsigned_v); + + struct CacheEntry { + Key key; + Value value; + PriorityType latest_use_tick_; + + bool operator<(const CacheEntry &rhs) const { + return latest_use_tick_ < rhs.latest_use_tick_; + } + }; + + SmallLruCache(size_t max_size) : kMaxSize{max_size} { + BOOST_ASSERT(kMaxSize > 0); + cache_.reserve(kMaxSize); + } + + std::optional> get(const Key &key) { + ticks_++; + if (ticks_ == 0) { + handleTicksOverflow(); + } + for (auto &entry : cache_) { + if (entry.key == key) { + entry.latest_use_tick_ = ticks_; + return entry.value; + } + } + return std::nullopt; + } + + template + void put(const Key &key, ValueArg &&value) { + static_assert(std::is_convertible_v< + ValueArg, + Value> || std::is_constructible_v); + ticks_++; + if (cache_.size() >= kMaxSize) { + auto min = std::min_element(cache_.begin(), cache_.end()); + cache_.erase(min); + } + cache_.push_back(CacheEntry{key, std::forward(value), ticks_}); + } + + private: + void handleTicksOverflow() { + // 'compress' timestamps of entries in the cache (works because we care + // only about their order, not actual timestamps) + std::sort(cache_.begin(), cache_.end()); + for (auto &entry : cache_) { + entry.latest_use_tick_ = ticks_; + ticks_++; + } + } + + const size_t kMaxSize; + // an abstract representation of time to implement 'recency' without + // depending on real time. Incremented on each cache access + PriorityType ticks_{}; + std::vector cache_; + }; + + /** + * @brief Pool of runtime instances - per state. Incapsulates modules cache. + * + */ + class RuntimeInstancesPool final + : public std::enable_shared_from_this { + using ModuleInstancePool = std::stack>; + + public: + using RootHash = storage::trie::RootHash; + using ModuleCache = + SmallLruCache>; + + /** + * @brief Instantiate new or reuse existing ModuleInstance for the provided + * state. + * + * @param state - the merkle trie root of the state containing the code of + * the runtime module we are acquiring an instance of. + * @return pointer to the acquired ModuleInstance if success. Error + * otherwise. + */ + outcome::result> tryAcquire( + const RootHash &state); + /** + * @brief Releases the module instance (returns it to the pool) + * + * @param state - the merkle trie root of the state containing the runtime + * module code we are releasing an instance of. + * @param instance - instance to be released. + */ + void release(const RootHash &state, + std::shared_ptr &&instance); + + /** + * @brief Get the module for state from internal cache + * + * @param state - the state containing the module's code. + * @return Module if any, nullopt otherwise + */ + std::optional> getModule(const RootHash &state); + + /** + * @brief Puts new module into internal cache + * + * @param state - runtime block, by its root hash + * @param module - new module pointer + */ + void putModule(const RootHash &state, std::shared_ptr module); + + private: + std::mutex mt_; + static constexpr size_t MODULES_CACHE_SIZE = 2; + ModuleCache modules_{MODULES_CACHE_SIZE}; + std::map pools_; + }; } // namespace kagome::runtime From aa1c4d5e8973cae3fbbd1495ec74cc2cd3e05ed0 Mon Sep 17 00:00:00 2001 From: Harrm Date: Wed, 26 Oct 2022 14:31:18 +0300 Subject: [PATCH 73/74] Revert offchain extension test --- core/runtime/common/runtime_environment_factory.cpp | 4 ++-- test/core/host_api/offchain_extension_test.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/runtime/common/runtime_environment_factory.cpp b/core/runtime/common/runtime_environment_factory.cpp index e21a53c524..ac20a1038b 100644 --- a/core/runtime/common/runtime_environment_factory.cpp +++ b/core/runtime/common/runtime_environment_factory.cpp @@ -89,7 +89,7 @@ namespace kagome::runtime { if (persistent_) { if (auto res = env.storage_provider->setToPersistentAt(storage_state_); !res) { - SL_ERROR(parent_factory->logger_, + SL_DEBUG(parent_factory->logger_, "Failed to set the storage state to hash {:l} when initializing a " "runtime environment; Reason: {}", storage_state_, @@ -99,7 +99,7 @@ namespace kagome::runtime { } else { if (auto res = env.storage_provider->setToEphemeralAt(storage_state_); !res) { - SL_ERROR(parent_factory->logger_, + SL_DEBUG(parent_factory->logger_, "Failed to set the storage state to hash {:l} when initializing a " "runtime environment; Reason: {}", storage_state_, diff --git a/test/core/host_api/offchain_extension_test.cpp b/test/core/host_api/offchain_extension_test.cpp index 00776ae634..59f4443c27 100644 --- a/test/core/host_api/offchain_extension_test.cpp +++ b/test/core/host_api/offchain_extension_test.cpp @@ -25,6 +25,7 @@ #include "testutil/prepare_loggers.hpp" using kagome::common::Buffer; +using kagome::common::BufferView; using kagome::host_api::OffchainExtension; using kagome::host_api::OffchainExtensionConfig; using kagome::offchain::Failure; @@ -298,14 +299,15 @@ TEST_P(TernaryParametrizedTest, LocalStorageCAS) { WasmPointer expected_pointer = 45; WasmSize expected_size = 45; WasmSpan expected_span = PtrSize(expected_pointer, expected_size).combine(); - Buffer expected(8, 'e'); + auto expected = Buffer{scale::encode(std::optional{}).value()}; EXPECT_CALL(*memory_, loadN(key_pointer, key_size)).WillOnce(Return(key)); EXPECT_CALL(*memory_, loadN(value_pointer, value_size)) .WillOnce(Return(value)); EXPECT_CALL(*memory_, loadN(expected_pointer, expected_size)) .WillOnce(Return(expected)); EXPECT_CALL(*offchain_worker_, - localStorageCompareAndSet(_, key.view(), _, value)) + localStorageCompareAndSet( + _, key.view(), std::optional{}, value)) .WillOnce(Return(true)); offchain_extension_->ext_offchain_local_storage_compare_and_set_version_1( GetParam(), key_span, expected_span, value_span); From 0c357b4f5f6ec4700eea26522a1de99f66dd61f5 Mon Sep 17 00:00:00 2001 From: Harrm Date: Thu, 27 Oct 2022 12:16:35 +0300 Subject: [PATCH 74/74] Reformat --- .../authority/impl/authority_manager_impl.cpp | 13 +++++++------ core/network/impl/stream_engine.hpp | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/core/consensus/authority/impl/authority_manager_impl.cpp b/core/consensus/authority/impl/authority_manager_impl.cpp index 038459fa82..6f4ce8f153 100644 --- a/core/consensus/authority/impl/authority_manager_impl.cpp +++ b/core/consensus/authority/impl/authority_manager_impl.cpp @@ -365,11 +365,10 @@ namespace kagome::authority { auto batch_res = trie_storage_->getEphemeralBatchAt(header.state_root); if (batch_res.has_error()) { if (batch_res.error() == storage::DatabaseError::NOT_FOUND) { - SL_DEBUG( - log_, - "Failed to fetch set id from trie storage: state {} is not in " - "the storage", - header.state_root); + SL_DEBUG(log_, + "Failed to fetch set id from trie storage: state {} is not in " + "the storage", + header.state_root); return std::nullopt; } return batch_res.as_failure(); @@ -916,7 +915,9 @@ namespace kagome::authority { while (ancestor->current_block != block) { bool goto_next_generation = false; for (const auto &node : ancestor->descendants) { - if (node->current_block == block) return node; + if (node->current_block == block) { + return node; + } if (directChainExists(node->current_block, block)) { ancestor = node; goto_next_generation = true; diff --git a/core/network/impl/stream_engine.hpp b/core/network/impl/stream_engine.hpp index b2df558050..69191bc5e6 100644 --- a/core/network/impl/stream_engine.hpp +++ b/core/network/impl/stream_engine.hpp @@ -523,9 +523,9 @@ namespace kagome::network { if (!stream_res) { SL_DEBUG(self->logger_, "Could not send message to new {} stream with {}: {}", - protocol->protocolName(), - peer_id, - stream_res.error().message()); + protocol->protocolName(), + peer_id, + stream_res.error().message()); if (stream_res == outcome::failure(