From ec8f324a3f144723eb2a3c5ba4c5b814a50b1363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 20 May 2024 18:39:45 +0200 Subject: [PATCH 01/10] Confirming set background --- nano/core_test/confirming_set.cpp | 8 +- nano/lib/stats_enums.hpp | 6 ++ nano/lib/thread_pool.hpp | 2 +- nano/lib/thread_roles.cpp | 3 + nano/lib/thread_roles.hpp | 1 + nano/node/confirming_set.cpp | 141 ++++++++++++++++++++---------- nano/node/confirming_set.hpp | 15 +++- nano/node/node.cpp | 2 +- 8 files changed, 124 insertions(+), 54 deletions(-) diff --git a/nano/core_test/confirming_set.cpp b/nano/core_test/confirming_set.cpp index 7b0df16c0c..f2fe621065 100644 --- a/nano/core_test/confirming_set.cpp +++ b/nano/core_test/confirming_set.cpp @@ -19,13 +19,13 @@ using namespace std::chrono_literals; TEST (confirming_set, construction) { auto ctx = nano::test::context::ledger_empty (); - nano::confirming_set confirming_set (ctx.ledger ()); + nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); } TEST (confirming_set, add_exists) { auto ctx = nano::test::context::ledger_send_receive (); - nano::confirming_set confirming_set (ctx.ledger ()); + nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); auto send = ctx.blocks ()[0]; confirming_set.add (send->hash ()); ASSERT_TRUE (confirming_set.exists (send->hash ())); @@ -34,7 +34,7 @@ TEST (confirming_set, add_exists) TEST (confirming_set, process_one) { auto ctx = nano::test::context::ledger_send_receive (); - nano::confirming_set confirming_set (ctx.ledger ()); + nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); std::atomic count = 0; std::mutex mutex; std::condition_variable condition; @@ -50,7 +50,7 @@ TEST (confirming_set, process_one) TEST (confirming_set, process_multiple) { auto ctx = nano::test::context::ledger_send_receive (); - nano::confirming_set confirming_set (ctx.ledger ()); + nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); std::atomic count = 0; std::mutex mutex; std::condition_variable condition; diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 27eb16bce4..ab491ab920 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -40,6 +40,7 @@ enum class type socket, confirmation_height, confirmation_observer, + confirming_set, drop, aggregator, requests, @@ -114,6 +115,8 @@ enum class detail rebroadcast, queue_overflow, triggered, + notify, + duplicate, // processing queue queue, @@ -440,6 +443,9 @@ enum class detail tier_2, tier_3, + // confirming_set + confirmed, + _last // Must be the last enum }; diff --git a/nano/lib/thread_pool.hpp b/nano/lib/thread_pool.hpp index f56a6271e0..b9f3607ed3 100644 --- a/nano/lib/thread_pool.hpp +++ b/nano/lib/thread_pool.hpp @@ -19,7 +19,7 @@ namespace nano class thread_pool final { public: - explicit thread_pool (unsigned, nano::thread_role::name); + explicit thread_pool (unsigned num_threads, nano::thread_role::name); ~thread_pool (); /** This will run when there is an available thread for execution */ diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index 76211682e4..98dc3b64e3 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -61,6 +61,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::confirmation_height_processing: thread_role_name_string = "Conf height"; break; + case nano::thread_role::name::confirmation_height_notifications: + thread_role_name_string = "Conf notif"; + break; case nano::thread_role::name::worker: thread_role_name_string = "Worker"; break; diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index d6d78b694f..b07026ab7a 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -25,6 +25,7 @@ enum class name rpc_request_processor, rpc_process_container, confirmation_height_processing, + confirmation_height_notifications, worker, bootstrap_worker, request_aggregator, diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index d39cf5791d..9aa101fe90 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -1,3 +1,5 @@ +#include "node.hpp" + #include #include #include @@ -5,9 +7,11 @@ #include #include -nano::confirming_set::confirming_set (nano::ledger & ledger, std::chrono::milliseconds batch_time) : +nano::confirming_set::confirming_set (nano::ledger & ledger, nano::stats & stats, std::chrono::milliseconds batch_time) : ledger{ ledger }, - batch_time{ batch_time } + stats{ stats }, + batch_time{ batch_time }, + workers{ 1, nano::thread_role::name::confirmation_height_notifications } { } @@ -18,14 +22,29 @@ nano::confirming_set::~confirming_set () void nano::confirming_set::add (nano::block_hash const & hash) { - std::lock_guard lock{ mutex }; - set.insert (hash); - condition.notify_all (); + bool added = false; + { + std::lock_guard lock{ mutex }; + auto [it, inserted] = set.insert (hash); + added = inserted; + } + if (added) + { + condition.notify_all (); + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::insert); + } + else + { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::duplicate); + } } void nano::confirming_set::start () { - thread = std::thread{ [this] () { run (); } }; + thread = std::thread{ [this] () { + nano::thread_role::set (nano::thread_role::name::confirmation_height_processing); + run (); + } }; } void nano::confirming_set::stop () @@ -39,6 +58,7 @@ void nano::confirming_set::stop () { thread.join (); } + workers.stop (); } bool nano::confirming_set::exists (nano::block_hash const & hash) const @@ -55,58 +75,86 @@ std::size_t nano::confirming_set::size () const void nano::confirming_set::run () { - nano::thread_role::set (nano::thread_role::name::confirmation_height_processing); std::unique_lock lock{ mutex }; - // Run the confirmation loop until stopped while (!stopped) { - condition.wait (lock, [&] () { return !set.empty () || stopped; }); - // Loop if there are items to process - if (!stopped && !set.empty ()) + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::loop); + + if (!set.empty ()) { - std::deque> cemented; - std::deque already; - // Move items in to back buffer and release lock so more items can be added to the front buffer - processing = std::move (this->set); - // Process all items in the back buffer - for (auto i = processing.begin (), n = processing.end (); !stopped && i != n;) - { - lock.unlock (); // Waiting for db write is potentially slow - auto guard = ledger.store.write_queue.wait (nano::store::writer::confirmation_height); - auto tx = ledger.tx_begin_write ({ nano::tables::confirmation_height }); - lock.lock (); - // Process items in the back buffer within a single transaction for a limited amount of time - for (auto timeout = std::chrono::steady_clock::now () + batch_time; !stopped && std::chrono::steady_clock::now () < timeout && i != n; ++i) - { - auto item = *i; - lock.unlock (); - auto added = ledger.confirm (tx, item); - if (!added.empty ()) - { - // Confirming this block may implicitly confirm more - cemented.insert (cemented.end (), added.begin (), added.end ()); - } - else - { - already.push_back (item); - } - lock.lock (); - } - } + run_batch (lock); + debug_assert (lock.owns_lock ()); + } + else + { + condition.wait (lock, [&] () { return !set.empty () || stopped; }); + } + } +} + +void nano::confirming_set::run_batch (std::unique_lock & lock) +{ + debug_assert (lock.owns_lock ()); + debug_assert (!mutex.try_lock ()); + debug_assert (!set.empty ()); + + std::deque> cemented; + std::deque already; + + // Move items in to back buffer and release lock so more items can be added to the front buffer + release_assert (processing.empty ()); + swap (set, processing); + + // Process all items in the back buffer + for (auto i = processing.begin (), n = processing.end (); !stopped && i != n;) + { + lock.unlock (); // Waiting for db write is potentially slow + + auto guard = ledger.store.write_queue.wait (nano::store::writer::confirmation_height); + auto tx = ledger.tx_begin_write ({ nano::tables::confirmation_height }); + + lock.lock (); + // Process items in the back buffer within a single transaction for a limited amount of time + for (auto timeout = std::chrono::steady_clock::now () + batch_time; !stopped && std::chrono::steady_clock::now () < timeout && i != n; ++i) + { + auto item = *i; lock.unlock (); - for (auto const & i : cemented) + + auto added = ledger.confirm (tx, item); + if (!added.empty ()) { - cemented_observers.notify (i); + // Confirming this block may implicitly confirm more + cemented.insert (cemented.end (), added.begin (), added.end ()); + stats.add (nano::stat::type::confirming_set, nano::stat::detail::confirmed, added.size ()); } - for (auto const & i : already) + else { - block_already_cemented_observers.notify (i); + already.push_back (item); + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::already_confirmed); } + lock.lock (); - // Clear and free back buffer by re-initializing - processing = decltype (processing){}; } } + + lock.unlock (); + + workers.push_task ([this, cemented = std::move (cemented), already = std::move (already)] () { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); + + for (auto const & i : cemented) + { + cemented_observers.notify (i); + } + for (auto const & i : already) + { + block_already_cemented_observers.notify (i); + } + }); + + lock.lock (); + + processing.clear (); } std::unique_ptr nano::confirming_set::collect_container_info (std::string const & name) const @@ -116,5 +164,6 @@ std::unique_ptr nano::confirming_set::collect_co auto composite = std::make_unique (name); composite->add_component (std::make_unique (container_info{ "set", set.size (), sizeof (typename decltype (set)::value_type) })); composite->add_component (std::make_unique (container_info{ "processing", processing.size (), sizeof (typename decltype (processing)::value_type) })); + composite->add_component (std::make_unique (container_info{ "notifications", workers.num_queued_tasks (), sizeof (std::function) })); return composite; } diff --git a/nano/node/confirming_set.hpp b/nano/node/confirming_set.hpp index 06feb52e11..c991d156ef 100644 --- a/nano/node/confirming_set.hpp +++ b/nano/node/confirming_set.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -11,8 +12,10 @@ namespace nano { +class node; class block; class ledger; +class stats; } namespace nano @@ -26,8 +29,9 @@ class confirming_set final friend class confirmation_height_pruned_source_Test; public: - confirming_set (nano::ledger & ledger, std::chrono::milliseconds batch_time = std::chrono::milliseconds{ 500 }); + confirming_set (nano::ledger &, nano::stats &, std::chrono::milliseconds batch_time = std::chrono::milliseconds{ 500 }); ~confirming_set (); + // Adds a block to the set of blocks to be confirmed void add (nano::block_hash const & hash); void start (); @@ -43,10 +47,17 @@ class confirming_set final private: void run (); + void run_batch (std::unique_lock &); + nano::ledger & ledger; - std::chrono::milliseconds batch_time; + nano::stats & stats; + + std::chrono::milliseconds const batch_time; std::unordered_set set; std::unordered_set processing; + + nano::thread_pool workers; + bool stopped{ false }; mutable std::mutex mutex; std::condition_variable condition; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 2391274812..9c220cacff 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -184,7 +184,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy application_path (application_path_a), port_mapping (*this), block_processor (*this), - confirming_set_impl{ std::make_unique (ledger, config.confirming_set_batch_time) }, + confirming_set_impl{ std::make_unique (ledger, stats, config.confirming_set_batch_time) }, confirming_set{ *confirming_set_impl }, active_impl{ std::make_unique (*this, confirming_set, block_processor) }, active{ *active_impl }, From b6ee1692c5d6f7833ac0a84fdfc94e5a57c3060b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 10:35:46 +0200 Subject: [PATCH 02/10] Batch cemented callback --- nano/lib/stats_enums.hpp | 5 ++++- nano/node/active_elections.cpp | 30 +++++++++++++++----------- nano/node/active_elections.hpp | 4 ++-- nano/node/confirming_set.cpp | 39 +++++++++++++++++++++------------- nano/node/confirming_set.hpp | 8 +++++++ nano/node/node.cpp | 1 + 6 files changed, 57 insertions(+), 30 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index ab491ab920..7ae41defd8 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -117,6 +117,8 @@ enum class detail triggered, notify, duplicate, + confirmed, + cemented, // processing queue queue, @@ -444,7 +446,8 @@ enum class detail tier_3, // confirming_set - confirmed, + notify_cemented, + notify_already_confirmed, _last // Must be the last enum }; diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index 780b22d39c..ba6ec95480 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -18,10 +18,10 @@ using namespace std::chrono; -nano::active_elections::active_elections (nano::node & node_a, nano::confirming_set & confirming_set, nano::block_processor & block_processor_a) : +nano::active_elections::active_elections (nano::node & node_a, nano::confirming_set & confirming_set_a, nano::block_processor & block_processor_a) : config{ node_a.config.active_elections }, node{ node_a }, - confirming_set{ confirming_set }, + confirming_set{ confirming_set_a }, block_processor{ block_processor_a }, recently_confirmed{ config.confirmation_cache }, recently_cemented{ config.confirmation_history_size }, @@ -29,14 +29,20 @@ nano::active_elections::active_elections (nano::node & node_a, nano::confirming_ { count_by_behavior.fill (0); // Zero initialize array - // Register a callback which will get called after a block is cemented - confirming_set.cemented_observers.add ([this] (std::shared_ptr const & callback_block_a) { - this->block_cemented_callback (callback_block_a); - }); + confirming_set.batch_cemented.add ([this] (nano::confirming_set::cemented_notification const & notification) { + { + auto transaction = node.ledger.tx_begin_read (); + for (auto const & block : notification.cemented) + { + transaction.refresh_if_needed (); - // Register a callback which will get called if a block is already cemented - confirming_set.block_already_cemented_observers.add ([this] (nano::block_hash const & hash_a) { - this->block_already_cemented_callback (hash_a); + block_cemented_callback (transaction, block); + } + } + for (auto const & hash : notification.already_cemented) + { + block_already_cemented_callback (hash); + } }); // Notify elections about alternative (forked) blocks @@ -84,7 +90,7 @@ void nano::active_elections::stop () clear (); } -void nano::active_elections::block_cemented_callback (std::shared_ptr const & block) +void nano::active_elections::block_cemented_callback (nano::secure::transaction const & transaction, std::shared_ptr const & block) { debug_assert (node.block_confirmed (block->hash ())); if (auto election_l = election (block->qualified_root ())) @@ -100,7 +106,7 @@ void nano::active_elections::block_cemented_callback (std::shared_ptrget_status (); votes = election->votes_with_weight (); } - if (confirming_set.exists (block->hash ())) + if (confirming_set.exists (block->hash ())) // TODO: This can be passed from the confirming_set { status.type = nano::election_status_type::active_confirmed_quorum; } @@ -113,7 +119,7 @@ void nano::active_elections::block_cemented_callback (std::shared_ptr= node.ledger.bootstrap_weight_max_blocks; bool was_active = status.type == nano::election_status_type::active_confirmed_quorum || status.type == nano::election_status_type::active_confirmation_height; diff --git a/nano/node/active_elections.hpp b/nano/node/active_elections.hpp index 46506698b5..0311740702 100644 --- a/nano/node/active_elections.hpp +++ b/nano/node/active_elections.hpp @@ -120,8 +120,6 @@ class active_elections final bool empty () const; std::size_t size () const; bool publish (std::shared_ptr const &); - void block_cemented_callback (std::shared_ptr const &); - void block_already_cemented_callback (nano::block_hash const &); /** * Maximum number of elections that should be present in this container @@ -148,6 +146,8 @@ class active_elections final std::vector> list_active_impl (std::size_t) const; void activate_successors (nano::secure::transaction const &, std::shared_ptr const & block); void notify_observers (nano::secure::transaction const &, nano::election_status const & status, std::vector const & votes) const; + void block_cemented_callback (nano::secure::transaction const &, std::shared_ptr const &); + void block_already_cemented_callback (nano::block_hash const &); private: // Dependencies active_elections_config const & config; diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index 9aa101fe90..c161ab8f50 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -7,12 +7,24 @@ #include #include -nano::confirming_set::confirming_set (nano::ledger & ledger, nano::stats & stats, std::chrono::milliseconds batch_time) : - ledger{ ledger }, - stats{ stats }, - batch_time{ batch_time }, +nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & stats_a, std::chrono::milliseconds batch_time_a) : + ledger{ ledger_a }, + stats{ stats_a }, + batch_time{ batch_time_a }, workers{ 1, nano::thread_role::name::confirmation_height_notifications } { + batch_cemented.add ([this] (auto const & notification) { + for (auto const & i : notification.cemented) + { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_cemented); + cemented_observers.notify (i); + } + for (auto const & i : notification.already_cemented) + { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_already_confirmed); + block_already_cemented_observers.notify (i); + } + }); } nano::confirming_set::~confirming_set () @@ -125,7 +137,7 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) { // Confirming this block may implicitly confirm more cemented.insert (cemented.end (), added.begin (), added.end ()); - stats.add (nano::stat::type::confirming_set, nano::stat::detail::confirmed, added.size ()); + stats.add (nano::stat::type::confirming_set, nano::stat::detail::cemented, added.size ()); } else { @@ -139,17 +151,14 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) lock.unlock (); - workers.push_task ([this, cemented = std::move (cemented), already = std::move (already)] () { - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); + cemented_notification notification{ + .cemented = std::move (cemented), + .already_cemented = std::move (already) + }; - for (auto const & i : cemented) - { - cemented_observers.notify (i); - } - for (auto const & i : already) - { - block_already_cemented_observers.notify (i); - } + workers.push_task ([this, notification = std::move (notification)] () { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); + batch_cemented.notify (notification); }); lock.lock (); diff --git a/nano/node/confirming_set.hpp b/nano/node/confirming_set.hpp index c991d156ef..3602c1325c 100644 --- a/nano/node/confirming_set.hpp +++ b/nano/node/confirming_set.hpp @@ -41,7 +41,15 @@ class confirming_set final std::size_t size () const; std::unique_ptr collect_container_info (std::string const & name) const; +public: // Events // Observers will be called once ledger has blocks marked as confirmed + struct cemented_notification + { + std::deque> cemented; + std::deque already_cemented; + }; + + nano::observer_set batch_cemented; nano::observer_set> cemented_observers; nano::observer_set block_already_cemented_observers; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 9c220cacff..0f8e9abb5e 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -476,6 +476,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy } } confirming_set.cemented_observers.add ([this] (auto const & block) { + // TODO: Is it neccessary to call this for all blocks? if (block->is_send ()) { workers.push_task ([this, hash = block->hash (), destination = block->destination ()] () { From 76e062e171473a3d0a5f00790be48c5aeab6a2ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 11:55:32 +0200 Subject: [PATCH 03/10] Use wallet workers --- nano/lib/thread_roles.cpp | 3 +++ nano/lib/thread_roles.hpp | 1 + nano/node/node.cpp | 7 ++++++- nano/node/node.hpp | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index 98dc3b64e3..fbb292f91d 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -70,6 +70,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::bootstrap_worker: thread_role_name_string = "Bootstrap work"; break; + case nano::thread_role::name::wallet_worker: + thread_role_name_string = "Wallet work"; + break; case nano::thread_role::name::request_aggregator: thread_role_name_string = "Req aggregator"; break; diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index b07026ab7a..818c36185b 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -28,6 +28,7 @@ enum class name confirmation_height_notifications, worker, bootstrap_worker, + wallet_worker, request_aggregator, state_block_signature_verification, epoch_upgrader, diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 0f8e9abb5e..eb5163897c 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -152,6 +152,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy stats{ logger, config.stats_config }, workers{ config.background_threads, nano::thread_role::name::worker }, bootstrap_workers{ config.bootstrap_serving_threads, nano::thread_role::name::bootstrap_worker }, + wallet_workers{ 1, nano::thread_role::name::wallet_worker }, flags (flags_a), work (work_a), distributed_work (*this), @@ -479,7 +480,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy // TODO: Is it neccessary to call this for all blocks? if (block->is_send ()) { - workers.push_task ([this, hash = block->hash (), destination = block->destination ()] () { + wallet_workers.push_task ([this, hash = block->hash (), destination = block->destination ()] () { wallets.receive_confirmed (hash, destination); }); } @@ -572,6 +573,8 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (collect_container_info (node.network, "network")); composite->add_component (node.telemetry.collect_container_info ("telemetry")); composite->add_component (collect_container_info (node.workers, "workers")); + composite->add_component (collect_container_info (node.bootstrap_workers, "bootstrap_workers")); + composite->add_component (collect_container_info (node.wallet_workers, "wallet_workers")); composite->add_component (collect_container_info (node.observers, "observers")); composite->add_component (collect_container_info (node.wallets, "wallets")); composite->add_component (node.vote_processor.collect_container_info ("vote_processor")); @@ -728,6 +731,8 @@ void nano::node::stop () logger.info (nano::log::type::node, "Node stopping..."); + bootstrap_workers.stop (); + wallet_workers.stop (); vote_router.stop (); peer_history.stop (); // Cancels ongoing work generation tasks, which may be blocking other threads diff --git a/nano/node/node.hpp b/nano/node/node.hpp index f63ba5ec47..2cdbe9d6c5 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -155,6 +155,7 @@ class node final : public std::enable_shared_from_this nano::stats stats; nano::thread_pool workers; nano::thread_pool bootstrap_workers; + nano::thread_pool wallet_workers; nano::node_flags flags; nano::work_pool & work; nano::distributed_work_factory distributed_work; From a0410919628b799b424fda76708418f3baf83d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 12:33:55 +0200 Subject: [PATCH 04/10] Stats --- nano/lib/logging_enums.hpp | 1 + nano/lib/stats_enums.hpp | 20 ++++++- nano/node/active_elections.cpp | 98 +++++++++++++++++++++++++++++----- nano/node/active_elections.hpp | 4 ++ nano/node/confirming_set.cpp | 2 +- nano/node/election_status.hpp | 3 ++ 6 files changed, 112 insertions(+), 16 deletions(-) diff --git a/nano/lib/logging_enums.hpp b/nano/lib/logging_enums.hpp index f78371b799..780d2b01ba 100644 --- a/nano/lib/logging_enums.hpp +++ b/nano/lib/logging_enums.hpp @@ -108,6 +108,7 @@ enum class detail // active_elections active_started, active_stopped, + active_cemented, // election election_confirmed, diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 7ae41defd8..355f426e60 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -62,9 +62,11 @@ enum class type active, active_elections, active_started, + active_stopped, active_confirmed, active_dropped, active_timeout, + active_cemented, backlog, unchecked, election_scheduler, @@ -378,6 +380,10 @@ enum class detail insert, insert_failed, + // active_elections + started, + stopped, + // unchecked put, satisfied, @@ -447,7 +453,19 @@ enum class detail // confirming_set notify_cemented, - notify_already_confirmed, + notify_already_cemented, + + // election_state + passive, + active, + expired_confirmed, + expired_unconfirmed, + + // election_status_type + ongoing, + active_confirmed_quorum, + active_confirmation_height, + inactive_confirmation_height, _last // Must be the last enum }; diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index ba6ec95480..31927cb734 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -120,7 +120,13 @@ void nano::active_elections::block_cemented_callback (nano::secure::transaction } recently_cemented.put (status); + node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::cemented); + node.stats.inc (nano::stat::type::active_cemented, to_stat_detail (status.type)); + + node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_cemented, nano::log::arg{ "election", election }); + notify_observers (transaction, status, votes); + bool cemented_bootstrap_count_reached = node.ledger.cemented_count () >= node.ledger.bootstrap_weight_max_blocks; bool was_active = status.type == nano::election_status_type::active_confirmed_quorum || status.type == nano::election_status_type::active_confirmation_height; @@ -302,7 +308,10 @@ void nano::active_elections::cleanup_election (nano::unique_lock & roots.get ().erase (roots.get ().find (election->qualified_root)); - node.stats.inc (completion_type (*election), to_stat_detail (election->behavior ())); + node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::stopped); + node.stats.inc (nano::stat::type::active_stopped, to_stat_detail (election->state ())); + node.stats.inc (to_stat_type (election->state ()), to_stat_detail (election->behavior ())); + node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_stopped, nano::log::arg{ "election", election }); node.logger.debug (nano::log::type::active_elections, "Erased election for blocks: {} (behavior: {}, state: {})", @@ -332,19 +341,6 @@ void nano::active_elections::cleanup_election (nano::unique_lock & } } -nano::stat::type nano::active_elections::completion_type (nano::election const & election) const -{ - if (election.confirmed ()) - { - return nano::stat::type::active_confirmed; - } - if (election.failed ()) - { - return nano::stat::type::active_timeout; - } - return nano::stat::type::active_dropped; -} - std::vector> nano::active_elections::list_active (std::size_t max_a) { nano::lock_guard guard{ mutex }; @@ -421,7 +417,9 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr< debug_assert (count_by_behavior[result.election->behavior ()] >= 0); count_by_behavior[result.election->behavior ()]++; + node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::started); node.stats.inc (nano::stat::type::active_started, to_stat_detail (election_behavior_a)); + node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_started, nano::log::arg{ "behavior", election_behavior_a }, nano::log::arg{ "election", result.election }); @@ -598,3 +596,75 @@ nano::error nano::active_elections_config::deserialize (nano::tomlconfig & toml) return toml.get_error (); } + +/* + * + */ + +nano::stat::type nano::to_stat_type (nano::election_state state) +{ + switch (state) + { + case election_state::passive: + case election_state::active: + return nano::stat::type::active_dropped; + break; + case election_state::confirmed: + case election_state::expired_confirmed: + return nano::stat::type::active_confirmed; + break; + case election_state::expired_unconfirmed: + return nano::stat::type::active_timeout; + break; + } + debug_assert (false); + return {}; +} + +nano::stat::detail nano::to_stat_detail (nano::election_state state) +{ + switch (state) + { + case election_state::passive: + return nano::stat::detail::passive; + break; + case election_state::active: + return nano::stat::detail::active; + break; + case election_state::confirmed: + return nano::stat::detail::confirmed; + break; + case election_state::expired_confirmed: + return nano::stat::detail::expired_confirmed; + break; + case election_state::expired_unconfirmed: + return nano::stat::detail::expired_unconfirmed; + break; + } + debug_assert (false); + return {}; +} + +nano::stat::detail nano::to_stat_detail (nano::election_status_type type) +{ + switch (type) + { + case election_status_type::ongoing: + return nano::stat::detail::ongoing; + break; + case election_status_type::active_confirmed_quorum: + return nano::stat::detail::active_confirmed_quorum; + break; + case election_status_type::active_confirmation_height: + return nano::stat::detail::active_confirmation_height; + break; + case election_status_type::inactive_confirmation_height: + return nano::stat::detail::inactive_confirmation_height; + break; + case election_status_type::stopped: + return nano::stat::detail::stopped; + break; + } + debug_assert (false); + return {}; +} \ No newline at end of file diff --git a/nano/node/active_elections.hpp b/nano/node/active_elections.hpp index 0311740702..ba40bb9d1a 100644 --- a/nano/node/active_elections.hpp +++ b/nano/node/active_elections.hpp @@ -36,6 +36,7 @@ class confirming_set; class election; class vote; class stats; +enum class election_state; } namespace nano::secure { @@ -197,4 +198,7 @@ class active_elections final }; std::unique_ptr collect_container_info (active_elections & active_elections, std::string const & name); + +nano::stat::type to_stat_type (nano::election_state); +nano::stat::detail to_stat_detail (nano::election_state); } diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index c161ab8f50..b7f675bf8c 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -21,7 +21,7 @@ nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & sta } for (auto const & i : notification.already_cemented) { - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_already_confirmed); + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_already_cemented); block_already_cemented_observers.notify (i); } }); diff --git a/nano/node/election_status.hpp b/nano/node/election_status.hpp index 014a14ad73..548bf8e2bf 100644 --- a/nano/node/election_status.hpp +++ b/nano/node/election_status.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -22,6 +23,8 @@ enum class election_status_type : uint8_t stopped = 5 }; +nano::stat::detail to_stat_detail (election_status_type); + /* Holds a summary of an election */ class election_status final { From 132c4232074f8948ea879f9acc84b58423a7d423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 12:43:13 +0200 Subject: [PATCH 05/10] Dedicated election workers --- nano/lib/stats_enums.hpp | 1 + nano/lib/thread_roles.cpp | 3 +++ nano/lib/thread_roles.hpp | 1 + nano/node/active_elections.cpp | 1 + nano/node/election.cpp | 2 +- nano/node/node.cpp | 5 ++++- nano/node/node.hpp | 1 + 7 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 355f426e60..0b13ea05ee 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -120,6 +120,7 @@ enum class detail notify, duplicate, confirmed, + unconfirmed, cemented, // processing queue diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index fbb292f91d..3fc2e3a54c 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -73,6 +73,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::wallet_worker: thread_role_name_string = "Wallet work"; break; + case nano::thread_role::name::election_worker: + thread_role_name_string = "Election work"; + break; case nano::thread_role::name::request_aggregator: thread_role_name_string = "Req aggregator"; break; diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index 818c36185b..e7009887fd 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -29,6 +29,7 @@ enum class name worker, bootstrap_worker, wallet_worker, + election_worker, request_aggregator, state_block_signature_verification, epoch_upgrader, diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index 31927cb734..21e0d12ce0 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -309,6 +309,7 @@ void nano::active_elections::cleanup_election (nano::unique_lock & roots.get ().erase (roots.get ().find (election->qualified_root)); node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::stopped); + node.stats.inc (nano::stat::type::active_elections, election->confirmed () ? nano::stat::detail::confirmed : nano::stat::detail::unconfirmed); node.stats.inc (nano::stat::type::active_stopped, to_stat_detail (election->state ())); node.stats.inc (to_stat_type (election->state ()), to_stat_detail (election->behavior ())); diff --git a/nano/node/election.cpp b/nano/node/election.cpp index 8e4854db2e..8ac2fc5628 100644 --- a/nano/node/election.cpp +++ b/nano/node/election.cpp @@ -63,7 +63,7 @@ void nano::election::confirm_once (nano::unique_lock & lock_a) lock_a.unlock (); - node.workers.push_task ([node_l = node.shared (), status_l, confirmation_action_l = confirmation_action] () { + node.election_workers.push_task ([node_l = node.shared (), status_l, confirmation_action_l = confirmation_action] () { node_l->process_confirmed (status_l); if (confirmation_action_l) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index eb5163897c..26a852fd09 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -153,6 +153,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy workers{ config.background_threads, nano::thread_role::name::worker }, bootstrap_workers{ config.bootstrap_serving_threads, nano::thread_role::name::bootstrap_worker }, wallet_workers{ 1, nano::thread_role::name::wallet_worker }, + election_workers{ 1, nano::thread_role::name::election_worker }, flags (flags_a), work (work_a), distributed_work (*this), @@ -575,6 +576,7 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (collect_container_info (node.workers, "workers")); composite->add_component (collect_container_info (node.bootstrap_workers, "bootstrap_workers")); composite->add_component (collect_container_info (node.wallet_workers, "wallet_workers")); + composite->add_component (collect_container_info (node.election_workers, "election_workers")); composite->add_component (collect_container_info (node.observers, "observers")); composite->add_component (collect_container_info (node.wallets, "wallets")); composite->add_component (node.vote_processor.collect_container_info ("vote_processor")); @@ -733,6 +735,7 @@ void nano::node::stop () bootstrap_workers.stop (); wallet_workers.stop (); + election_workers.stop (); vote_router.stop (); peer_history.stop (); // Cancels ongoing work generation tasks, which may be blocking other threads @@ -1248,7 +1251,7 @@ void nano::node::process_confirmed (nano::election_status const & status_a, uint { iteration_a++; std::weak_ptr node_w (shared ()); - workers.add_timed_task (std::chrono::steady_clock::now () + network_params.node.process_confirmed_interval, [node_w, status_a, iteration_a] () { + election_workers.add_timed_task (std::chrono::steady_clock::now () + network_params.node.process_confirmed_interval, [node_w, status_a, iteration_a] () { if (auto node_l = node_w.lock ()) { node_l->process_confirmed (status_a, iteration_a); diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 2cdbe9d6c5..994bf1dce3 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -156,6 +156,7 @@ class node final : public std::enable_shared_from_this nano::thread_pool workers; nano::thread_pool bootstrap_workers; nano::thread_pool wallet_workers; + nano::thread_pool election_workers; nano::node_flags flags; nano::work_pool & work; nano::distributed_work_factory distributed_work; From c470c885a9beb28ece6698ea3b1bab96e00774ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 15:24:38 +0200 Subject: [PATCH 06/10] Pass confirmation root info --- nano/node/active_elections.cpp | 8 ++++---- nano/node/active_elections.hpp | 4 ++-- nano/node/confirming_set.cpp | 16 ++++++++++------ nano/node/confirming_set.hpp | 4 +++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index 21e0d12ce0..7efde7a794 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -32,11 +32,11 @@ nano::active_elections::active_elections (nano::node & node_a, nano::confirming_ confirming_set.batch_cemented.add ([this] (nano::confirming_set::cemented_notification const & notification) { { auto transaction = node.ledger.tx_begin_read (); - for (auto const & block : notification.cemented) + for (auto const & [block, confirmation_root] : notification.cemented) { transaction.refresh_if_needed (); - block_cemented_callback (transaction, block); + block_cemented_callback (transaction, block, confirmation_root); } } for (auto const & hash : notification.already_cemented) @@ -90,7 +90,7 @@ void nano::active_elections::stop () clear (); } -void nano::active_elections::block_cemented_callback (nano::secure::transaction const & transaction, std::shared_ptr const & block) +void nano::active_elections::block_cemented_callback (nano::secure::transaction const & transaction, std::shared_ptr const & block, nano::block_hash const & confirmation_root) { debug_assert (node.block_confirmed (block->hash ())); if (auto election_l = election (block->qualified_root ())) @@ -106,7 +106,7 @@ void nano::active_elections::block_cemented_callback (nano::secure::transaction status = election->get_status (); votes = election->votes_with_weight (); } - if (confirming_set.exists (block->hash ())) // TODO: This can be passed from the confirming_set + if (block->hash () == confirmation_root) { status.type = nano::election_status_type::active_confirmed_quorum; } diff --git a/nano/node/active_elections.hpp b/nano/node/active_elections.hpp index ba40bb9d1a..a4cb478a38 100644 --- a/nano/node/active_elections.hpp +++ b/nano/node/active_elections.hpp @@ -147,8 +147,8 @@ class active_elections final std::vector> list_active_impl (std::size_t) const; void activate_successors (nano::secure::transaction const &, std::shared_ptr const & block); void notify_observers (nano::secure::transaction const &, nano::election_status const & status, std::vector const & votes) const; - void block_cemented_callback (nano::secure::transaction const &, std::shared_ptr const &); - void block_already_cemented_callback (nano::block_hash const &); + void block_cemented_callback (nano::secure::transaction const &, std::shared_ptr const & block, nano::block_hash const & confirmation_root); + void block_already_cemented_callback (nano::block_hash const & hash); private: // Dependencies active_elections_config const & config; diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index b7f675bf8c..d27d05e66c 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -14,15 +14,15 @@ nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & sta workers{ 1, nano::thread_role::name::confirmation_height_notifications } { batch_cemented.add ([this] (auto const & notification) { - for (auto const & i : notification.cemented) + for (auto const & [block, confirmation_root] : notification.cemented) { stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_cemented); - cemented_observers.notify (i); + cemented_observers.notify (block); } - for (auto const & i : notification.already_cemented) + for (auto const & hash : notification.already_cemented) { stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_already_cemented); - block_already_cemented_observers.notify (i); + block_already_cemented_observers.notify (hash); } }); } @@ -110,7 +110,7 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) debug_assert (!mutex.try_lock ()); debug_assert (!set.empty ()); - std::deque> cemented; + std::deque cemented; std::deque already; // Move items in to back buffer and release lock so more items can be added to the front buffer @@ -136,7 +136,11 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) if (!added.empty ()) { // Confirming this block may implicitly confirm more - cemented.insert (cemented.end (), added.begin (), added.end ()); + for (auto & block : added) + { + cemented.emplace_back (block, item); + } + stats.add (nano::stat::type::confirming_set, nano::stat::detail::cemented, added.size ()); } else diff --git a/nano/node/confirming_set.hpp b/nano/node/confirming_set.hpp index 3602c1325c..0619c700d3 100644 --- a/nano/node/confirming_set.hpp +++ b/nano/node/confirming_set.hpp @@ -43,9 +43,11 @@ class confirming_set final public: // Events // Observers will be called once ledger has blocks marked as confirmed + using cemented_t = std::pair, nano::block_hash>; // + struct cemented_notification { - std::deque> cemented; + std::deque cemented; std::deque already_cemented; }; From 04bb841ba0b17467fa199611daca24e6dff64ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 15:32:52 +0200 Subject: [PATCH 07/10] More stats work --- nano/core_test/active_elections.cpp | 6 ++-- nano/lib/stats_enums.hpp | 12 +++---- nano/node/active_elections.cpp | 55 +++++------------------------ 3 files changed, 18 insertions(+), 55 deletions(-) diff --git a/nano/core_test/active_elections.cpp b/nano/core_test/active_elections.cpp index 692bcae8fc..d0755d3019 100644 --- a/nano/core_test/active_elections.cpp +++ b/nano/core_test/active_elections.cpp @@ -666,7 +666,7 @@ TEST (active_elections, dropped_cleanup) ASSERT_FALSE (node.network.publish_filter.apply (block_bytes.data (), block_bytes.size ())); // An election was recently dropped - ASSERT_EQ (1, node.stats.count (nano::stat::type::active_dropped, nano::stat::detail::manual)); + ASSERT_EQ (1, node.stats.count (nano::stat::type::active_elections_dropped, nano::stat::detail::manual)); // Block cleared from active ASSERT_FALSE (node.vote_router.active (hash)); @@ -684,7 +684,7 @@ TEST (active_elections, dropped_cleanup) ASSERT_TRUE (node.network.publish_filter.apply (block_bytes.data (), block_bytes.size ())); // Not dropped - ASSERT_EQ (1, node.stats.count (nano::stat::type::active_dropped, nano::stat::detail::manual)); + ASSERT_EQ (1, node.stats.count (nano::stat::type::active_elections_dropped, nano::stat::detail::manual)); // Block cleared from active ASSERT_FALSE (node.vote_router.active (hash)); @@ -1387,7 +1387,7 @@ TEST (active_elections, limit_vote_hinted_elections) ASSERT_TIMELY (5s, nano::test::active (node, { open1 })); // Ensure there was no overflow of elections - ASSERT_EQ (0, node.stats.count (nano::stat::type::active_dropped, nano::stat::detail::priority)); + ASSERT_EQ (0, node.stats.count (nano::stat::type::active_elections_dropped, nano::stat::detail::priority)); } /* diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 0b13ea05ee..4fdc489992 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -61,12 +61,12 @@ enum class type bootstrap_server_response, active, active_elections, - active_started, - active_stopped, - active_confirmed, - active_dropped, - active_timeout, - active_cemented, + active_elections_started, + active_elections_stopped, + active_elections_confirmed, + active_elections_dropped, + active_elections_timeout, + active_elections_cemented, backlog, unchecked, election_scheduler, diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index 7efde7a794..d1f1c22496 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -121,7 +122,7 @@ void nano::active_elections::block_cemented_callback (nano::secure::transaction recently_cemented.put (status); node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::cemented); - node.stats.inc (nano::stat::type::active_cemented, to_stat_detail (status.type)); + node.stats.inc (nano::stat::type::active_elections_cemented, to_stat_detail (status.type)); node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_cemented, nano::log::arg{ "election", election }); @@ -310,7 +311,7 @@ void nano::active_elections::cleanup_election (nano::unique_lock & node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::stopped); node.stats.inc (nano::stat::type::active_elections, election->confirmed () ? nano::stat::detail::confirmed : nano::stat::detail::unconfirmed); - node.stats.inc (nano::stat::type::active_stopped, to_stat_detail (election->state ())); + node.stats.inc (nano::stat::type::active_elections_stopped, to_stat_detail (election->state ())); node.stats.inc (to_stat_type (election->state ()), to_stat_detail (election->behavior ())); node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_stopped, nano::log::arg{ "election", election }); @@ -419,7 +420,7 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr< count_by_behavior[result.election->behavior ()]++; node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::started); - node.stats.inc (nano::stat::type::active_started, to_stat_detail (election_behavior_a)); + node.stats.inc (nano::stat::type::active_elections_started, to_stat_detail (election_behavior_a)); node.logger.trace (nano::log::type::active_elections, nano::log::detail::active_started, nano::log::arg{ "behavior", election_behavior_a }, @@ -608,14 +609,14 @@ nano::stat::type nano::to_stat_type (nano::election_state state) { case election_state::passive: case election_state::active: - return nano::stat::type::active_dropped; + return nano::stat::type::active_elections_dropped; break; case election_state::confirmed: case election_state::expired_confirmed: - return nano::stat::type::active_confirmed; + return nano::stat::type::active_elections_confirmed; break; case election_state::expired_unconfirmed: - return nano::stat::type::active_timeout; + return nano::stat::type::active_elections_timeout; break; } debug_assert (false); @@ -624,48 +625,10 @@ nano::stat::type nano::to_stat_type (nano::election_state state) nano::stat::detail nano::to_stat_detail (nano::election_state state) { - switch (state) - { - case election_state::passive: - return nano::stat::detail::passive; - break; - case election_state::active: - return nano::stat::detail::active; - break; - case election_state::confirmed: - return nano::stat::detail::confirmed; - break; - case election_state::expired_confirmed: - return nano::stat::detail::expired_confirmed; - break; - case election_state::expired_unconfirmed: - return nano::stat::detail::expired_unconfirmed; - break; - } - debug_assert (false); - return {}; + return nano::enum_util::cast (state); } nano::stat::detail nano::to_stat_detail (nano::election_status_type type) { - switch (type) - { - case election_status_type::ongoing: - return nano::stat::detail::ongoing; - break; - case election_status_type::active_confirmed_quorum: - return nano::stat::detail::active_confirmed_quorum; - break; - case election_status_type::active_confirmation_height: - return nano::stat::detail::active_confirmation_height; - break; - case election_status_type::inactive_confirmation_height: - return nano::stat::detail::inactive_confirmation_height; - break; - case election_status_type::stopped: - return nano::stat::detail::stopped; - break; - } - debug_assert (false); - return {}; + return nano::enum_util::cast (type); } \ No newline at end of file From 48fb94bf46da0b4d7e6e4c2125f51fcdef0295ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 15:46:40 +0200 Subject: [PATCH 08/10] Collect container info --- nano/lib/thread_pool.cpp | 4 ++-- nano/lib/thread_pool.hpp | 4 ++-- nano/node/confirming_set.cpp | 8 ++++---- nano/node/confirming_set.hpp | 3 ++- nano/node/node.cpp | 8 ++++---- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/nano/lib/thread_pool.cpp b/nano/lib/thread_pool.cpp index 571a840718..d3e3ed140e 100644 --- a/nano/lib/thread_pool.cpp +++ b/nano/lib/thread_pool.cpp @@ -89,9 +89,9 @@ void nano::thread_pool::set_thread_names (nano::thread_role::name thread_name) thread_names_latch.wait (); } -std::unique_ptr nano::collect_container_info (thread_pool & thread_pool, std::string const & name) +std::unique_ptr nano::thread_pool::collect_container_info (std::string const & name) const { auto composite = std::make_unique (name); - composite->add_component (std::make_unique (container_info{ "count", thread_pool.num_queued_tasks (), sizeof (std::function) })); + composite->add_component (std::make_unique (container_info{ "count", num_queued_tasks (), sizeof (std::function) })); return composite; } diff --git a/nano/lib/thread_pool.hpp b/nano/lib/thread_pool.hpp index b9f3607ed3..b8eb29f9c0 100644 --- a/nano/lib/thread_pool.hpp +++ b/nano/lib/thread_pool.hpp @@ -37,6 +37,8 @@ class thread_pool final /** Returns the number of tasks which are awaiting execution by the thread pool **/ uint64_t num_queued_tasks () const; + std::unique_ptr collect_container_info (std::string const & name) const; + private: nano::mutex mutex; std::atomic stopped{ false }; @@ -48,6 +50,4 @@ class thread_pool final std::latch thread_names_latch; void set_thread_names (nano::thread_role::name thread_name); }; - -std::unique_ptr collect_container_info (thread_pool & thread_pool, std::string const & name); } // namespace nano diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index d27d05e66c..b9202a07d1 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -11,7 +11,7 @@ nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & sta ledger{ ledger_a }, stats{ stats_a }, batch_time{ batch_time_a }, - workers{ 1, nano::thread_role::name::confirmation_height_notifications } + notification_workers{ 1, nano::thread_role::name::confirmation_height_notifications } { batch_cemented.add ([this] (auto const & notification) { for (auto const & [block, confirmation_root] : notification.cemented) @@ -70,7 +70,7 @@ void nano::confirming_set::stop () { thread.join (); } - workers.stop (); + notification_workers.stop (); } bool nano::confirming_set::exists (nano::block_hash const & hash) const @@ -160,7 +160,7 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) .already_cemented = std::move (already) }; - workers.push_task ([this, notification = std::move (notification)] () { + notification_workers.push_task ([this, notification = std::move (notification)] () { stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); batch_cemented.notify (notification); }); @@ -177,6 +177,6 @@ std::unique_ptr nano::confirming_set::collect_co auto composite = std::make_unique (name); composite->add_component (std::make_unique (container_info{ "set", set.size (), sizeof (typename decltype (set)::value_type) })); composite->add_component (std::make_unique (container_info{ "processing", processing.size (), sizeof (typename decltype (processing)::value_type) })); - composite->add_component (std::make_unique (container_info{ "notifications", workers.num_queued_tasks (), sizeof (std::function) })); + composite->add_component (notification_workers.collect_container_info ("notification_workers")); return composite; } diff --git a/nano/node/confirming_set.hpp b/nano/node/confirming_set.hpp index 0619c700d3..41261112a8 100644 --- a/nano/node/confirming_set.hpp +++ b/nano/node/confirming_set.hpp @@ -39,6 +39,7 @@ class confirming_set final // Added blocks will remain in this set until after ledger has them marked as confirmed. bool exists (nano::block_hash const & hash) const; std::size_t size () const; + std::unique_ptr collect_container_info (std::string const & name) const; public: // Events @@ -66,7 +67,7 @@ class confirming_set final std::unordered_set set; std::unordered_set processing; - nano::thread_pool workers; + nano::thread_pool notification_workers; bool stopped{ false }; mutable std::mutex mutex; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 26a852fd09..a11aa54ff8 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -573,10 +573,10 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (node.tcp_listener.collect_container_info ("tcp_listener")); composite->add_component (collect_container_info (node.network, "network")); composite->add_component (node.telemetry.collect_container_info ("telemetry")); - composite->add_component (collect_container_info (node.workers, "workers")); - composite->add_component (collect_container_info (node.bootstrap_workers, "bootstrap_workers")); - composite->add_component (collect_container_info (node.wallet_workers, "wallet_workers")); - composite->add_component (collect_container_info (node.election_workers, "election_workers")); + composite->add_component (node.workers.collect_container_info ("workers")); + composite->add_component (node.bootstrap_workers.collect_container_info ("bootstrap_workers")); + composite->add_component (node.wallet_workers.collect_container_info ("wallet_workers")); + composite->add_component (node.election_workers.collect_container_info ("election_workers")); composite->add_component (collect_container_info (node.observers, "observers")); composite->add_component (collect_container_info (node.wallets, "wallets")); composite->add_component (node.vote_processor.collect_container_info ("vote_processor")); From 4204f980ff8bfb10f70ad5c63687a7e61606b7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 24 May 2024 15:47:46 +0200 Subject: [PATCH 09/10] Fixes --- nano/lib/stats_enums.hpp | 1 + nano/node/confirming_set.cpp | 8 ++++---- nano/slow_test/node.cpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 4fdc489992..6bec768d2e 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -455,6 +455,7 @@ enum class detail // confirming_set notify_cemented, notify_already_cemented, + already_cemented, // election_state passive, diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index b9202a07d1..075bd71789 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -1,5 +1,3 @@ -#include "node.hpp" - #include #include #include @@ -53,6 +51,8 @@ void nano::confirming_set::add (nano::block_hash const & hash) void nano::confirming_set::start () { + debug_assert (!thread.joinable ()); + thread = std::thread{ [this] () { nano::thread_role::set (nano::thread_role::name::confirmation_height_processing); run (); @@ -146,7 +146,7 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) else { already.push_back (item); - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::already_confirmed); + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::already_cemented); } lock.lock (); @@ -167,7 +167,7 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) lock.lock (); - processing.clear (); + processing = {}; // Avoid permamently holding memory if the set was large } std::unique_ptr nano::confirming_set::collect_container_info (std::string const & name) const diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 4d6f906936..960a205a64 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -1145,7 +1145,7 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) nano::block_hash block_hash_being_processed{ 0 }; nano::store::write_queue write_queue{ false }; - nano::confirming_set confirming_set{ ledger }; + nano::confirming_set confirming_set{ ledger, stats }; auto const num_accounts = 100000; From ea84af93169a20e7df3cebd4d37d6fe4fcd2d32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sun, 2 Jun 2024 12:50:44 +0200 Subject: [PATCH 10/10] Proper batch sizes for confirming set --- nano/node/active_elections.cpp | 1 + nano/node/confirming_set.cpp | 58 ++++++++++++++++------------------ nano/node/confirming_set.hpp | 5 ++- nano/node/node.cpp | 2 +- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index d1f1c22496..92bfde942f 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -94,6 +94,7 @@ void nano::active_elections::stop () void nano::active_elections::block_cemented_callback (nano::secure::transaction const & transaction, std::shared_ptr const & block, nano::block_hash const & confirmation_root) { debug_assert (node.block_confirmed (block->hash ())); + if (auto election_l = election (block->qualified_root ())) { election_l->try_confirm (block->hash ()); diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index 075bd71789..b577e3d5e4 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -5,10 +5,9 @@ #include #include -nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & stats_a, std::chrono::milliseconds batch_time_a) : +nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & stats_a) : ledger{ ledger_a }, stats{ stats_a }, - batch_time{ batch_time_a }, notification_workers{ 1, nano::thread_role::name::confirmation_height_notifications } { batch_cemented.add ([this] (auto const & notification) { @@ -76,13 +75,13 @@ void nano::confirming_set::stop () bool nano::confirming_set::exists (nano::block_hash const & hash) const { std::lock_guard lock{ mutex }; - return set.count (hash) != 0 || processing.count (hash) != 0; + return set.count (hash) != 0; } std::size_t nano::confirming_set::size () const { std::lock_guard lock{ mutex }; - return set.size () + processing.size (); + return set.size (); } void nano::confirming_set::run () @@ -95,7 +94,8 @@ void nano::confirming_set::run () if (!set.empty ()) { run_batch (lock); - debug_assert (lock.owns_lock ()); + debug_assert (!lock.owns_lock ()); + lock.lock (); } else { @@ -104,6 +104,21 @@ void nano::confirming_set::run () } } +std::deque nano::confirming_set::next_batch (size_t max_count) +{ + debug_assert (!mutex.try_lock ()); + debug_assert (!set.empty ()); + + std::deque results; + while (!set.empty () && results.size () < max_count) + { + auto it = set.begin (); + results.push_back (*it); + set.erase (it); + } + return results; +} + void nano::confirming_set::run_batch (std::unique_lock & lock) { debug_assert (lock.owns_lock ()); @@ -113,48 +128,36 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) std::deque cemented; std::deque already; - // Move items in to back buffer and release lock so more items can be added to the front buffer - release_assert (processing.empty ()); - swap (set, processing); + auto batch = next_batch (256); - // Process all items in the back buffer - for (auto i = processing.begin (), n = processing.end (); !stopped && i != n;) - { - lock.unlock (); // Waiting for db write is potentially slow + lock.unlock (); + { + // TODO: Properly limiting batch times requires this combo to be wrapped in a single object that provides refresh functionality auto guard = ledger.store.write_queue.wait (nano::store::writer::confirmation_height); auto tx = ledger.tx_begin_write ({ nano::tables::confirmation_height }); - lock.lock (); - // Process items in the back buffer within a single transaction for a limited amount of time - for (auto timeout = std::chrono::steady_clock::now () + batch_time; !stopped && std::chrono::steady_clock::now () < timeout && i != n; ++i) + for (auto const & hash : batch) { - auto item = *i; - lock.unlock (); - - auto added = ledger.confirm (tx, item); + auto added = ledger.confirm (tx, hash); if (!added.empty ()) { // Confirming this block may implicitly confirm more for (auto & block : added) { - cemented.emplace_back (block, item); + cemented.emplace_back (block, hash); } stats.add (nano::stat::type::confirming_set, nano::stat::detail::cemented, added.size ()); } else { - already.push_back (item); + already.push_back (hash); stats.inc (nano::stat::type::confirming_set, nano::stat::detail::already_cemented); } - - lock.lock (); } } - lock.unlock (); - cemented_notification notification{ .cemented = std::move (cemented), .already_cemented = std::move (already) @@ -164,10 +167,6 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); batch_cemented.notify (notification); }); - - lock.lock (); - - processing = {}; // Avoid permamently holding memory if the set was large } std::unique_ptr nano::confirming_set::collect_container_info (std::string const & name) const @@ -176,7 +175,6 @@ std::unique_ptr nano::confirming_set::collect_co auto composite = std::make_unique (name); composite->add_component (std::make_unique (container_info{ "set", set.size (), sizeof (typename decltype (set)::value_type) })); - composite->add_component (std::make_unique (container_info{ "processing", processing.size (), sizeof (typename decltype (processing)::value_type) })); composite->add_component (notification_workers.collect_container_info ("notification_workers")); return composite; } diff --git a/nano/node/confirming_set.hpp b/nano/node/confirming_set.hpp index 41261112a8..d6b8350de4 100644 --- a/nano/node/confirming_set.hpp +++ b/nano/node/confirming_set.hpp @@ -29,7 +29,7 @@ class confirming_set final friend class confirmation_height_pruned_source_Test; public: - confirming_set (nano::ledger &, nano::stats &, std::chrono::milliseconds batch_time = std::chrono::milliseconds{ 500 }); + confirming_set (nano::ledger &, nano::stats &); ~confirming_set (); // Adds a block to the set of blocks to be confirmed @@ -59,13 +59,12 @@ class confirming_set final private: void run (); void run_batch (std::unique_lock &); + std::deque next_batch (size_t max_count); nano::ledger & ledger; nano::stats & stats; - std::chrono::milliseconds const batch_time; std::unordered_set set; - std::unordered_set processing; nano::thread_pool notification_workers; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index a11aa54ff8..33f07fe417 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -186,7 +186,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy application_path (application_path_a), port_mapping (*this), block_processor (*this), - confirming_set_impl{ std::make_unique (ledger, stats, config.confirming_set_batch_time) }, + confirming_set_impl{ std::make_unique (ledger, stats) }, confirming_set{ *confirming_set_impl }, active_impl{ std::make_unique (*this, confirming_set, block_processor) }, active{ *active_impl },