Sequential elections

Building off the multi-level election controls added in #2785, this PR changes the block processing behavior to only start an election if the block's ancestors are confirmed.

- There is a superlinear usage of resources the more elections are active for the same chain, which is now prevented.
- On confirming an election, the successors (in the same chain and destination chain for sends) are activated (functionality from #2785).
- Frontier confirmation works as the fallback behavior, for bootstraps and nodes restarting after getting blocks. Backtracking (#2778) later starts the bottom-most election if unable to confirm the frontier.

nano/core_test/active_transactions.cpp

@@ -115,6 +115,7 @@ TEST (active_transactions, adjusted_multiplier_priority)
 	node1.process_active (send2); // genesis
 	node1.process_active (open1); // key1
 	node1.process_active (open2); // key2
+	nano::blocks_confirm (node1, { send1, send2, open1, open2 });
 	system.deadline_set (10s);
 	while (node1.active.size () != 4)
 	{
@@ -162,6 +163,7 @@ TEST (active_transactions, adjusted_multiplier_priority)
 	node1.process_active (send4); // genesis
 	node1.process_active (send6); // key1
 	node1.process_active (send8); // key2
+	nano::blocks_confirm (node1, { send3, send4, send5, send6, send7, send8 });
 
 	system.deadline_set (10s);
 	while (node1.active.size () != 6)
@@ -265,6 +267,7 @@ TEST (active_transactions, prioritize_chains)
 	node1.process_active (send1);
 	node1.process_active (open1);
 	node1.process_active (send5);
+	nano::blocks_confirm (node1, { send1, open1, send5 });
 	system.deadline_set (10s);
 	while (node1.active.size () != 3)
 	{
@@ -282,7 +285,7 @@ TEST (active_transactions, prioritize_chains)
 	node1.process_active (send3);
 	node1.process_active (send4);
 	node1.process_active (send6);
-
+	nano::blocks_confirm (node1, { send2, send3, send4, send6 });
 	system.deadline_set (10s);
 	while (node1.active.size () != 4)
 	{
@@ -551,7 +554,7 @@ TEST (active_transactions, vote_replays)
 	ASSERT_NE (nullptr, open1);
 	node.process_active (send1);
 	node.process_active (open1);
-	node.block_processor.flush ();
+	nano::blocks_confirm (node, { send1, open1 });
 	ASSERT_EQ (2, node.active.size ());
 	// First vote is not a replay and confirms the election, second vote should be a replay since the election has confirmed but not yet removed
 	auto vote_send1 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 0, send1));
@@ -573,7 +576,7 @@ TEST (active_transactions, vote_replays)
 	auto send2 (std::make_shared<nano::state_block> (key.pub, open1->hash (), key.pub, nano::Gxrb_ratio - 1, key.pub, key.prv, key.pub, *system.work.generate (open1->hash ())));
 	ASSERT_NE (nullptr, send2);
 	node.process_active (send2);
-	node.block_processor.flush ();
+	nano::blocks_confirm (node, { send2 });
 	ASSERT_EQ (1, node.active.size ());
 	auto vote1_send2 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 0, send2));
 	auto vote2_send2 (std::make_shared<nano::vote> (key.pub, key.prv, 0, send2));
@@ -660,6 +663,7 @@ TEST (active_transactions, activate_dependencies)
 	                                      .build ();
 	node2->process_active (block2);
 	node2->block_processor.flush ();
+	node2->block_confirm (block2);
 	system.deadline_set (10s);
 	while (node1->block (block2->hash ()) == nullptr)
 	{
@@ -991,6 +995,19 @@ TEST (active_transactions, election_difficulty_update_fork)
 	ASSERT_EQ (nano::process_result::progress, node.process (*send1).code);
 	ASSERT_EQ (nano::process_result::progress, node.process (*open1).code);
 	ASSERT_EQ (nano::process_result::progress, node.process (*send2).code);
+	// Confirm blocks so far to allow starting elections for upcoming blocks
+	for (auto block : { open1, send2 })
+	{
+		node.block_confirm (block);
+		{
+			auto election = node.active.election (block->qualified_root ());
+			ASSERT_NE (nullptr, election);
+			nano::lock_guard<std::mutex> guard (node.active.mutex);
+			election->confirm_once ();
+		}
+		ASSERT_TIMELY (2s, node.block_confirmed (block->hash ()));
+		node.active.erase (*block);
+	}
 
 	// Verify an election with multiple blocks is correctly updated on arrival of another block
 	// Each subsequent block has difficulty at least higher than the previous one

nano/core_test/confirmation_height.cpp

@@ -154,6 +154,7 @@ TEST (confirmation_height, multiple_accounts)
 		auto receive3 = std::make_shared<nano::receive_block> (open3.hash (), send6.hash (), key3.prv, key3.pub, *system.work.generate (open3.hash ()));
 		node->process_active (receive3);
 		node->block_processor.flush ();
+		node->block_confirm (receive3);
 		{
 			auto election = node->active.election (receive3->qualified_root ());
 			ASSERT_NE (nullptr, election);
@@ -358,6 +359,7 @@ TEST (confirmation_height, gap_live)
 		// Now complete the chain where the block comes in on the live network
 		node->process_active (open1);
 		node->block_processor.flush ();
+		node->block_confirm (open1);
 		{
 			auto election = node->active.election (open1->qualified_root ());
 			ASSERT_NE (nullptr, election);
@@ -447,6 +449,7 @@ TEST (confirmation_height, send_receive_between_2_accounts)
 
 		node->process_active (receive4);
 		node->block_processor.flush ();
+		node->block_confirm (receive4);
 		{
 			auto election = node->active.election (receive4->qualified_root ());
 			ASSERT_NE (nullptr, election);
@@ -1237,7 +1240,7 @@ TEST (confirmation_height, callback_confirmed_history)
 
 		node->process_active (send1);
 		node->block_processor.flush ();
-
+		node->block_confirm (send1);
 		{
 			node->process_active (send);
 			node->block_processor.flush ();

nano/core_test/conflicts.cpp

@@ -219,8 +219,10 @@ TEST (conflicts, dependency)
 
 TEST (conflicts, adjusted_multiplier)
 {
-	nano::system system (1);
-	auto & node1 (*system.nodes[0]);
+	nano::system system;
+	nano::node_flags flags;
+	flags.disable_request_loop = true;
+	auto & node1 (*system.add_node (flags));
 	nano::genesis genesis;
 	nano::keypair key1;
 	nano::keypair key2;
@@ -250,7 +252,7 @@ TEST (conflicts, adjusted_multiplier)
 	nano::keypair key4;
 	auto send5 (std::make_shared<nano::state_block> (key3.pub, change1->hash (), nano::test_genesis_key.pub, 0, key4.pub, key3.prv, key3.pub, *system.work.generate (change1->hash ()))); // Pending for open epoch block
 	node1.process_active (send5);
-	node1.block_processor.flush ();
+	nano::blocks_confirm (node1, { send1, send2, receive1, open1, send3, send4, open_epoch1, receive2, open2, change1, send5 });
 	system.deadline_set (3s);
 	while (node1.active.size () != 11)
 	{
@@ -285,6 +287,7 @@ TEST (conflicts, adjusted_multiplier)
 	ASSERT_GT (open_epoch2->difficulty (), nano::difficulty::from_multiplier ((adjusted_multipliers.find (send1->hash ())->second), node1.network_params.network.publish_thresholds.base));
 	node1.process_active (open_epoch2);
 	node1.block_processor.flush ();
+	node1.block_confirm (open_epoch2);
 	system.deadline_set (3s);
 	while (node1.active.size () != 12)
 	{

nano/core_test/gap_cache.cpp

@@ -63,38 +63,38 @@ TEST (gap_cache, comparison)
 	ASSERT_EQ (arrival, cache.blocks.get<1> ().begin ()->arrival);
 }
 
+// Upon receiving enough votes for a gapped block, a lazy bootstrap should be initiated
 TEST (gap_cache, gap_bootstrap)
 {
-	nano::system system (2);
+	nano::node_flags node_flags;
+	node_flags.disable_legacy_bootstrap = true;
+	node_flags.disable_request_loop = true; // to avoid fallback behavior of broadcasting blocks
+	nano::system system (2, nano::transport::transport_type::tcp, node_flags);
+
 	auto & node1 (*system.nodes[0]);
 	auto & node2 (*system.nodes[1]);
 	nano::block_hash latest (node1.latest (nano::test_genesis_key.pub));
 	nano::keypair key;
 	auto send (std::make_shared<nano::send_block> (latest, key.pub, nano::genesis_amount - 100, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (latest)));
-	{
-		nano::block_post_events events;
-		auto transaction (node1.store.tx_begin_write ());
-		ASSERT_EQ (nano::process_result::progress, node1.block_processor.process_one (transaction, events, send).code);
-	}
+	node1.process (*send);
 	ASSERT_EQ (nano::genesis_amount - 100, node1.balance (nano::genesis_account));
 	ASSERT_EQ (nano::genesis_amount, node2.balance (nano::genesis_account));
+	// Confirm send block, allowing voting on the upcoming block
+	node1.block_confirm (send);
+	{
+		auto election = node1.active.election (send->qualified_root ());
+		ASSERT_NE (nullptr, election);
+		nano::lock_guard<std::mutex> guard (node1.active.mutex);
+		election->confirm_once ();
+	}
+	ASSERT_TIMELY (2s, node1.block_confirmed (send->hash ()));
+	node1.active.erase (*send);
 	system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv);
-	system.wallet (0)->insert_adhoc (key.prv);
 	auto latest_block (system.wallet (0)->send_action (nano::test_genesis_key.pub, key.pub, 100));
 	ASSERT_NE (nullptr, latest_block);
 	ASSERT_EQ (nano::genesis_amount - 200, node1.balance (nano::genesis_account));
 	ASSERT_EQ (nano::genesis_amount, node2.balance (nano::genesis_account));
-	system.deadline_set (10s);
-	{
-		// The separate publish and vote system doesn't work very well here because it's instantly confirmed.
-		// We help it get the block and vote out here.
-		auto transaction (node1.store.tx_begin_read ());
-		node1.network.flood_block (latest_block);
-	}
-	while (node2.balance (nano::genesis_account) != nano::genesis_amount - 200)
-	{
-		ASSERT_NO_ERROR (system.poll ());
-	}
+	ASSERT_TIMELY (10s, node2.balance (nano::genesis_account) == nano::genesis_amount - 200);
 }
 
 TEST (gap_cache, two_dependencies)

nano/core_test/ledger.cpp

@@ -930,8 +930,9 @@ TEST (votes, add_old_different_account)
 	node1.work_generate_blocking (*send1);
 	auto send2 (std::make_shared<nano::send_block> (send1->hash (), key1.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0));
 	node1.work_generate_blocking (*send2);
-	ASSERT_EQ (nano::process_result::progress, node1.process_local (send1).code);
-	ASSERT_EQ (nano::process_result::progress, node1.process_local (send2).code);
+	ASSERT_EQ (nano::process_result::progress, node1.process (*send1).code);
+	ASSERT_EQ (nano::process_result::progress, node1.process (*send2).code);
+	nano::blocks_confirm (node1, { send1, send2 });
 	auto election1 = node1.active.election (send1->qualified_root ());
 	ASSERT_NE (nullptr, election1);
 	auto election2 = node1.active.election (send2->qualified_root ());
@@ -2662,10 +2663,11 @@ TEST (ledger, block_hash_account_conflict)
 	node1.work_generate_blocking (*receive1);
 	node1.work_generate_blocking (*send2);
 	node1.work_generate_blocking (*open_epoch1);
-	ASSERT_EQ (nano::process_result::progress, node1.process_local (send1).code);
-	ASSERT_EQ (nano::process_result::progress, node1.process_local (receive1).code);
-	ASSERT_EQ (nano::process_result::progress, node1.process_local (send2).code);
-	ASSERT_EQ (nano::process_result::progress, node1.process_local (open_epoch1).code);
+	ASSERT_EQ (nano::process_result::progress, node1.process (*send1).code);
+	ASSERT_EQ (nano::process_result::progress, node1.process (*receive1).code);
+	ASSERT_EQ (nano::process_result::progress, node1.process (*send2).code);
+	ASSERT_EQ (nano::process_result::progress, node1.process (*open_epoch1).code);
+	nano::blocks_confirm (node1, { send1, receive1, send2, open_epoch1 });
 	auto election1 = node1.active.election (send1->qualified_root ());
 	ASSERT_NE (nullptr, election1);
 	auto election2 = node1.active.election (receive1->qualified_root ());