Benchmarking suite

.gitmodules

@@ -37,3 +37,6 @@
 [submodule "submodules/fmt"]
 	path = submodules/fmt
 	url = https://github.com/fmtlib/fmt.git
+[submodule "submodules/benchmark"]
+	path = submodules/benchmark
+	url = https://github.com/google/benchmark.git

CMakeLists.txt

@@ -737,10 +737,15 @@ if(NANO_TEST OR RAIBLOCKS_TEST)
     gtest PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
                      "${CMAKE_SOURCE_DIR}/submodules/gtest/googletest/include")
 
+  set(BENCHMARK_ENABLE_TESTING OFF)
+  add_subdirectory(submodules/benchmark)
+
   add_subdirectory(nano/test_common)
   add_subdirectory(nano/core_test)
   add_subdirectory(nano/rpc_test)
   add_subdirectory(nano/slow_test)
+  add_subdirectory(nano/benchmarks)
+
   add_custom_target(
     all_tests
     COMMAND echo "BATCH BUILDING TESTS"

nano/benchmarks/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_executable(benchmarks entry.cpp ledger.cpp stats.cpp)
+
+target_link_libraries(benchmarks test_common benchmark::benchmark)
+
+include_directories(${CMAKE_SOURCE_DIR}/submodules)

nano/benchmarks/entry.cpp

@@ -0,0 +1,20 @@
+#include <benchmark/benchmark.h>
+
+static void BM_StringCreation (benchmark::State & state)
+{
+	for (auto _ : state)
+		std::string empty_string;
+}
+// Register the function as a benchmark
+BENCHMARK (BM_StringCreation);
+
+// Define another benchmark
+static void BM_StringCopy (benchmark::State & state)
+{
+	std::string x = "hello";
+	for (auto _ : state)
+		std::string copy (x);
+}
+BENCHMARK (BM_StringCopy);
+
+BENCHMARK_MAIN ();

nano/benchmarks/ledger.cpp

@@ -0,0 +1,105 @@
+#include <nano/lib/logging.hpp>
+#include <nano/node/make_store.hpp>
+#include <nano/node/nodeconfig.hpp>
+#include <nano/secure/ledger.hpp>
+#include <nano/secure/ledger_set_any.hpp>
+#include <nano/secure/utility.hpp>
+#include <nano/store/account.hpp>
+
+#include <benchmark/benchmark.h>
+
+// Expects live ledger in default location
+// PLEASE NOTE: Make sure to purge disk cache between runs (`purge` command on macOS)
+static void BM_ledger_iterate_accounts (benchmark::State & state)
+{
+	nano::logger::initialize_dummy ();
+	nano::logger logger;
+	nano::stats stats{ logger };
+
+	// Use live ledger
+	nano::networks network = nano::networks::nano_live_network;
+	nano::network_params network_params{ network };
+	nano::node_config config{ network_params };
+	config.rocksdb_config.enable = state.range (0);
+	auto application_path = nano::working_path (network);
+
+	auto store_impl{ nano::make_store (logger, application_path, network_params.ledger, false, true, config.rocksdb_config, config.diagnostics_config.txn_tracking, config.block_processor_batch_max_time, config.lmdb_config, config.backup_before_upgrade) };
+	auto & store{ *store_impl };
+
+	if (store.init_error ())
+	{
+		state.SkipWithError ("Store initialization failed. Make sure ledger files are present in the default location.");
+	}
+
+	auto ledger_impl{ std::make_unique<nano::ledger> (store, stats, network_params.ledger, nano::generate_cache_flags::all_disabled (), config.representative_vote_weight_minimum.number ()) };
+	auto & ledger{ *ledger_impl };
+
+	auto transaction = ledger.tx_begin_read ();
+	nano::account current{ 0 };
+	nano::account_info current_info;
+	auto it = ledger.any.account_begin (transaction);
+	for (auto _ : state)
+	{
+		if (it != ledger.any.account_end ())
+		{
+			current = it->first;
+			current_info = it->second;
+			benchmark::DoNotOptimize (current);
+			benchmark::DoNotOptimize (current_info);
+
+			++it;
+		}
+		else
+		{
+			break;
+		}
+	}
+}
+BENCHMARK (BM_ledger_iterate_accounts)->ArgName ("use_rocksdb")->Arg (0)->Arg (1);
+
+// Expects live ledger in default location
+// PLEASE NOTE: Make sure to purge disk cache between runs (`purge` command on macOS)
+static void BM_store_iterate_accounts (benchmark::State & state)
+{
+	nano::logger::initialize_dummy ();
+	nano::logger logger;
+	nano::stats stats{ logger };
+
+	// Use live ledger
+	nano::networks network = nano::networks::nano_live_network;
+	nano::network_params network_params{ network };
+	nano::node_config config{ network_params };
+	config.rocksdb_config.enable = state.range (0);
+	nano::node_flags flags;
+	auto application_path = nano::working_path (network);
+
+	auto store_impl{ nano::make_store (logger, application_path, network_params.ledger, false, true, config.rocksdb_config, config.diagnostics_config.txn_tracking, config.block_processor_batch_max_time, config.lmdb_config, config.backup_before_upgrade, flags.force_use_write_queue) };
+	auto & store{ *store_impl };
+
+	if (store.init_error ())
+	{
+		state.SkipWithError ("Store initialization failed. Make sure ledger files are present in the default location.");
+	}
+
+	auto transaction = store.tx_begin_read ();
+	nano::account current{ 0 };
+	nano::account_info current_info;
+	auto it = store.account.begin (transaction);
+	for (auto _ : state)
+	{
+		if (it != store.account.end ())
+		{
+			current = it->first;
+			current_info = it->second;
+			benchmark::DoNotOptimize (current);
+			benchmark::DoNotOptimize (current_info);
+
+			++it;
+		}
+		else
+		{
+			break;
+		}
+	}
+}
+BENCHMARK (BM_store_iterate_accounts)->ArgName ("use_rocksdb")->Arg (0)->Arg (1);

nano/benchmarks/stats.cpp

@@ -0,0 +1,49 @@
+#include <nano/lib/enum_util.hpp>
+#include <nano/lib/logging.hpp>
+#include <nano/lib/stats.hpp>
+
+#include <random>
+
+#include <benchmark/benchmark.h>
+
+static void BM_stats_inc_single (benchmark::State & state)
+{
+	nano::logger::initialize_dummy ();
+	nano::stats stats{ nano::default_logger () };
+
+	for (auto _ : state)
+	{
+		stats.inc (nano::stat::type::ledger, nano::stat::detail::open);
+	}
+}
+
+BENCHMARK (BM_stats_inc_single);
+BENCHMARK (BM_stats_inc_single)->Threads (10);
+
+static void BM_stats_inc_random (benchmark::State & state)
+{
+	nano::logger::initialize_dummy ();
+	nano::stats stats{ nano::default_logger () };
+
+	auto random_subset = [] (auto elements, size_t count) -> std::vector<typename decltype (elements)::value_type> {
+		std::shuffle (elements.begin (), elements.end (), std::mt19937 (std::random_device () ()));
+		return { elements.begin (), elements.begin () + std::min (count, elements.size ()) };
+	};
+
+	auto stat_types = random_subset (nano::enum_util::values<nano::stat::type> (), state.range (0));
+	auto stat_details = random_subset (nano::enum_util::values<nano::stat::detail> (), state.range (1));
+
+	size_t type_index = 0;
+	size_t detail_index = 0;
+
+	for (auto _ : state)
+	{
+		stats.inc (stat_types[type_index], stat_details[detail_index]);
+
+		type_index = (type_index + 1) % stat_types.size ();
+		detail_index = (detail_index + 1) % stat_details.size ();
+	}
+}
+
+BENCHMARK (BM_stats_inc_random)->Args ({ 32, 32 });
+BENCHMARK (BM_stats_inc_random)->Args ({ 32, 32 })->Threads (10);

nano/lib/logging.cpp

@@ -117,6 +117,18 @@ void nano::logger::initialize_for_tests (nano::log_config fallback)
 	global_initialized = true;
 }
 
+void nano::logger::initialize_dummy ()
+{
+	nano::log_config config{
+		.default_level = nano::log::level::off,
+		.flush_level = nano::log::level::off,
+		.console.enable = false,
+		.file.enable = false,
+	};
+	initialize_common (config, std::nullopt);
+	global_initialized = true;
+}
+
 // Using std::cerr here, since logging may not be initialized yet
 void nano::logger::initialize_common (nano::log_config const & config, std::optional<std::filesystem::path> data_path)
 {
@@ -307,7 +319,7 @@ nano::log_config nano::log_config::cli_default ()
 {
 	log_config config{};
 	config.default_level = nano::log::level::critical;
-	config.console.colors = false; // to avoid printing warning about cerr and colors
+	config.console.colors = false;
 	config.console.to_cerr = true; // Use cerr to avoid interference with CLI output that goes to stdout
 	config.file.enable = false;
 	return config;

nano/lib/logging.hpp

@@ -157,6 +157,7 @@ class logger final
 public:
 	static void initialize (nano::log_config fallback, std::optional<std::filesystem::path> data_path = std::nullopt, std::vector<std::string> const & config_overrides = {});
 	static void initialize_for_tests (nano::log_config fallback);
+	static void initialize_dummy (); // TODO: This is less than ideal, provide `nano::dummy_logger ()` instead
 	static void flush ();
 
 private:

nano/secure/generate_cache_flags.hpp

@@ -14,5 +14,17 @@ class generate_cache_flags
 	bool block_count = true;
 
 	void enable_all ();
+
+public:
+	static generate_cache_flags all_disabled ()
+	{
+		generate_cache_flags flags;
+		flags.reps = false;
+		flags.cemented_count = false;
+		flags.unchecked_count = false;
+		flags.account_count = false;
+		flags.block_count = false;
+		return flags;
+	}
 };
 }

nano/store/lmdb/lmdb.hpp

@@ -64,6 +64,7 @@ class component : public nano::store::component
 
 public:
 	component (nano::logger &, std::filesystem::path const &, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);
+
 	store::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_requiring_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) override;
 	store::read_transaction tx_begin_read () const override;