From 819c5a15a80847c4ad98b06b54e8ce33d0b53fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 6 May 2024 15:13:02 +0200 Subject: [PATCH 01/13] Move env utilities to `nano::env::...` --- nano/lib/config.cpp | 6 +++--- nano/lib/env.cpp | 6 +++--- nano/lib/env.hpp | 8 ++++---- nano/lib/logging.cpp | 8 +++----- nano/lib/stats.cpp | 2 +- nano/secure/utility.cpp | 4 ++-- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/nano/lib/config.cpp b/nano/lib/config.cpp index 293ab2f6c6..7f15dd041d 100644 --- a/nano/lib/config.cpp +++ b/nano/lib/config.cpp @@ -320,13 +320,13 @@ std::string get_tls_toml_config_path (std::filesystem::path const & data_path) std::string nano::get_env_or_default (char const * variable_name, std::string default_value) { - auto value = nano::get_env (variable_name); - return value ? *value : default_value; + auto value = nano::env::get (variable_name); + return value.value_or (default_value); } int nano::get_env_int_or_default (const char * variable_name, const int default_value) { - auto value = nano::get_env (variable_name); + auto value = nano::env::get (variable_name); if (value) { try diff --git a/nano/lib/env.cpp b/nano/lib/env.cpp index 80792e5f6a..43588533a9 100644 --- a/nano/lib/env.cpp +++ b/nano/lib/env.cpp @@ -4,7 +4,7 @@ #include -std::optional nano::get_env (std::string_view name) +std::optional nano::env::get (std::string_view name) { std::string name_str{ name }; if (auto value = std::getenv (name_str.c_str ())) @@ -14,12 +14,12 @@ std::optional nano::get_env (std::string_view name) return std::nullopt; } -std::optional nano::get_env_bool (std::string_view name) +std::optional nano::env::get_bool (std::string_view name) { std::vector const on_values{ "1", "true", "on" }; std::vector const off_values{ "0", "false", "off" }; - if (auto value = get_env (name)) + if (auto value = get (name)) { // Using case-insensitive comparison if (std::any_of (on_values.begin (), on_values.end (), [&value] (auto const & on) { return boost::iequals (*value, on); })) diff --git a/nano/lib/env.hpp b/nano/lib/env.hpp index 87263b6968..42969ab114 100644 --- a/nano/lib/env.hpp +++ b/nano/lib/env.hpp @@ -3,13 +3,13 @@ #include #include -namespace nano -{ /* * Get environment variable as a specific type or none if variable is not present. */ -std::optional get_env (std::string_view name); +namespace nano::env +{ +std::optional get (std::string_view name); // @throws std::invalid_argument if the value is not a valid boolean -std::optional get_env_bool (std::string_view name); +std::optional get_bool (std::string_view name); } \ No newline at end of file diff --git a/nano/lib/logging.cpp b/nano/lib/logging.cpp index da7f6e8dc3..0953007fa0 100644 --- a/nano/lib/logging.cpp +++ b/nano/lib/logging.cpp @@ -461,7 +461,7 @@ nano::log_config nano::load_log_config (nano::log_config fallback, const std::fi auto config = nano::load_config_file (fallback, config_filename, data_path, config_overrides); // Parse default log level from environment variable, e.g. "NANO_LOG=debug" - auto env_level = nano::get_env ("NANO_LOG"); + auto env_level = nano::env::get ("NANO_LOG"); if (env_level) { try @@ -478,8 +478,7 @@ nano::log_config nano::load_log_config (nano::log_config fallback, const std::fi } // Parse per logger levels from environment variable, e.g. "NANO_LOG_LEVELS=ledger=debug,node=trace" - auto env_levels = nano::get_env ("NANO_LOG_LEVELS"); - if (env_levels) + if (auto env_levels = nano::env::get ("NANO_LOG_LEVELS")) { std::map levels; for (auto const & env_level_str : nano::util::split (*env_levels, ",")) @@ -516,8 +515,7 @@ nano::log_config nano::load_log_config (nano::log_config fallback, const std::fi } } - auto env_tracing_format = nano::get_env ("NANO_TRACE_FORMAT"); - if (env_tracing_format) + if (auto env_tracing_format = nano::env::get ("NANO_TRACE_FORMAT")) { try { diff --git a/nano/lib/stats.cpp b/nano/lib/stats.cpp index 642c2ab3ed..310648cf48 100644 --- a/nano/lib/stats.cpp +++ b/nano/lib/stats.cpp @@ -33,7 +33,7 @@ std::string nano::stat_log_sink::tm_to_string (tm & tm) nano::stats::stats (nano::logger & logger_a, nano::stats_config config_a) : config{ std::move (config_a) }, logger{ logger_a }, - enable_logging{ nano::get_env_bool ("NANO_LOG_STATS").value_or (false) } + enable_logging{ nano::env::get_bool ("NANO_LOG_STATS").value_or (false) } { } diff --git a/nano/secure/utility.cpp b/nano/secure/utility.cpp index cb0f049a9a..9a8f4d6dfc 100644 --- a/nano/secure/utility.cpp +++ b/nano/secure/utility.cpp @@ -11,9 +11,9 @@ static std::vector all_unique_paths; std::filesystem::path nano::working_path (nano::networks network) { - auto result (nano::app_path ()); + auto result = nano::app_path (); - if (auto path_override = nano::get_env ("NANO_APP_PATH")) + if (auto path_override = nano::env::get ("NANO_APP_PATH")) { result = *path_override; std::cerr << "Application path overridden by NANO_APP_PATH environment variable: " << result << std::endl; From ff86186a9ea45a5a0eacc8625a6921c5ef5f4071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 6 May 2024 15:20:24 +0200 Subject: [PATCH 02/13] Env get_int helpers --- nano/lib/env.cpp | 32 ++++++++++++++++++++++++++++++++ nano/lib/env.hpp | 6 ++++++ 2 files changed, 38 insertions(+) diff --git a/nano/lib/env.cpp b/nano/lib/env.cpp index 43588533a9..3b67347c1d 100644 --- a/nano/lib/env.cpp +++ b/nano/lib/env.cpp @@ -34,4 +34,36 @@ std::optional nano::env::get_bool (std::string_view name) throw std::invalid_argument ("Invalid environment boolean value: " + *value); } return std::nullopt; +} + +std::optional nano::env::get_int (std::string_view name) +{ + if (auto value = get (name)) + { + try + { + return std::stoi (*value); + } + catch (std::invalid_argument const &) + { + throw std::invalid_argument ("Invalid environment integer value: " + *value); + } + } + return std::nullopt; +} + +std::optional nano::env::get_uint (std::string_view name) +{ + if (auto value = get (name)) + { + try + { + return std::stoul (*value); + } + catch (std::invalid_argument const &) + { + throw std::invalid_argument ("Invalid environment unsigned integer value: " + *value); + } + } + return std::nullopt; } \ No newline at end of file diff --git a/nano/lib/env.hpp b/nano/lib/env.hpp index 42969ab114..ccd911c21f 100644 --- a/nano/lib/env.hpp +++ b/nano/lib/env.hpp @@ -12,4 +12,10 @@ std::optional get (std::string_view name); // @throws std::invalid_argument if the value is not a valid boolean std::optional get_bool (std::string_view name); + +// @throws std::invalid_argument if the value is not a valid integer +std::optional get_int (std::string_view name); + +// @throws std::invalid_argument if the value is not a valid integer +std::optional get_uint (std::string_view name); } \ No newline at end of file From f54c7b5f63ee3ac62014aed9334800ae5f426cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 6 May 2024 15:24:19 +0200 Subject: [PATCH 03/13] Hardware concurrency override --- nano/lib/threading.cpp | 18 ++++++++++-------- nano/lib/threading.hpp | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index 6da0e000af..5fb6b5d3a7 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -15,16 +16,17 @@ boost::thread::attributes nano::thread_attributes::get_default () return attrs; } -unsigned int nano::hardware_concurrency () +unsigned nano::hardware_concurrency () { - // Try to read overridden value from environment variable - static int value = nano::get_env_int_or_default ("NANO_HARDWARE_CONCURRENCY", 0); - if (value <= 0) - { - // Not present or invalid, use default + static auto const concurrency = [] () { + if (auto value = nano::env::get_uint ("NANO_HARDWARE_CONCURRENCY")) + { + std::cerr << "Hardware concurrency overridden by NANO_HARDWARE_CONCURRENCY environment variable: " << *value << std::endl; + return *value; + } return std::thread::hardware_concurrency (); - } - return value; + }(); + return concurrency; } bool nano::join_or_pass (std::thread & thread) diff --git a/nano/lib/threading.hpp b/nano/lib/threading.hpp index 2877d8672a..dee7362890 100644 --- a/nano/lib/threading.hpp +++ b/nano/lib/threading.hpp @@ -16,7 +16,7 @@ namespace thread_attributes /** * Number of available logical processor cores. Might be overridden by setting `NANO_HARDWARE_CONCURRENCY` environment variable */ -unsigned int hardware_concurrency (); +unsigned hardware_concurrency (); /** * If thread is joinable joins it, otherwise does nothing From 67f8e91348413e2b4df10cd1be21198d6eba60e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 6 May 2024 15:27:21 +0200 Subject: [PATCH 04/13] App path override --- nano/secure/plat/osx/working.mm | 2 +- nano/secure/plat/posix/working.cpp | 2 +- nano/secure/plat/windows/working.cpp | 2 +- nano/secure/utility.cpp | 19 +++++++++++++------ nano/secure/utility.hpp | 1 + nano/secure/working.hpp | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/nano/secure/plat/osx/working.mm b/nano/secure/plat/osx/working.mm index cc977abe59..239985e92c 100644 --- a/nano/secure/plat/osx/working.mm +++ b/nano/secure/plat/osx/working.mm @@ -4,7 +4,7 @@ namespace nano { -std::filesystem::path app_path () +std::filesystem::path app_path_impl () { NSString * dir_string = [NSSearchPathForDirectoriesInDomains(NSLibraryDirectory, NSUserDomainMask, YES) lastObject]; char const * dir_chars = [dir_string UTF8String]; diff --git a/nano/secure/plat/posix/working.cpp b/nano/secure/plat/posix/working.cpp index 5f016fce92..97d33f768f 100644 --- a/nano/secure/plat/posix/working.cpp +++ b/nano/secure/plat/posix/working.cpp @@ -6,7 +6,7 @@ namespace nano { -std::filesystem::path app_path () +std::filesystem::path app_path_impl () { auto entry (getpwuid (getuid ())); debug_assert (entry != nullptr); diff --git a/nano/secure/plat/windows/working.cpp b/nano/secure/plat/windows/working.cpp index 2253272e6e..4654b7dfcc 100644 --- a/nano/secure/plat/windows/working.cpp +++ b/nano/secure/plat/windows/working.cpp @@ -4,7 +4,7 @@ namespace nano { -std::filesystem::path app_path () +std::filesystem::path app_path_impl () { std::filesystem::path result; WCHAR path[MAX_PATH]; diff --git a/nano/secure/utility.cpp b/nano/secure/utility.cpp index 9a8f4d6dfc..ac974d9f56 100644 --- a/nano/secure/utility.cpp +++ b/nano/secure/utility.cpp @@ -9,16 +9,23 @@ static std::vector all_unique_paths; +std::filesystem::path nano::app_path () +{ + static auto const path = [] () { + if (auto value = nano::env::get ("NANO_APP_PATH")) + { + std::cerr << "Application path overridden by NANO_APP_PATH environment variable: " << *value << std::endl; + return std::filesystem::path{ *value }; + } + return nano::app_path_impl (); + }(); + return path; +} + std::filesystem::path nano::working_path (nano::networks network) { auto result = nano::app_path (); - if (auto path_override = nano::env::get ("NANO_APP_PATH")) - { - result = *path_override; - std::cerr << "Application path overridden by NANO_APP_PATH environment variable: " << result << std::endl; - } - switch (network) { case nano::networks::invalid: diff --git a/nano/secure/utility.hpp b/nano/secure/utility.hpp index 8cd446c90a..abb70df840 100644 --- a/nano/secure/utility.hpp +++ b/nano/secure/utility.hpp @@ -6,6 +6,7 @@ namespace nano { +std::filesystem::path app_path (); // OS-specific way of finding a path to a home directory. std::filesystem::path working_path (nano::networks network = nano::network_constants::active_network); // Get a unique path within the home directory, used for testing. diff --git a/nano/secure/working.hpp b/nano/secure/working.hpp index fe32f4942b..dfb8218228 100644 --- a/nano/secure/working.hpp +++ b/nano/secure/working.hpp @@ -4,5 +4,5 @@ namespace nano { -std::filesystem::path app_path (); +std::filesystem::path app_path_impl (); } From 4ed66fe1f5592f650a08508922f79b76441b9f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 May 2024 20:35:56 +0200 Subject: [PATCH 05/13] Convert legacy calls to `nano::env::get (...)` --- nano/lib/config.cpp | 79 +++++++++++++++------------------------- nano/lib/config.hpp | 8 ---- nano/node/nodeconfig.cpp | 7 ++-- nano/secure/common.cpp | 6 ++- 4 files changed, 37 insertions(+), 63 deletions(-) diff --git a/nano/lib/config.cpp b/nano/lib/config.cpp index 7f15dd041d..70b136357f 100644 --- a/nano/lib/config.cpp +++ b/nano/lib/config.cpp @@ -238,35 +238,6 @@ uint64_t get_env_threshold_or_default (char const * variable_name, uint64_t cons return value ? boost::lexical_cast> (value) : default_value; } -uint16_t test_node_port () -{ - auto test_env = nano::get_env_or_default ("NANO_TEST_NODE_PORT", "17075"); - return boost::lexical_cast (test_env); -} -uint16_t test_rpc_port () -{ - auto test_env = nano::get_env_or_default ("NANO_TEST_RPC_PORT", "17076"); - return boost::lexical_cast (test_env); -} -uint16_t test_ipc_port () -{ - auto test_env = nano::get_env_or_default ("NANO_TEST_IPC_PORT", "17077"); - return boost::lexical_cast (test_env); -} -uint16_t test_websocket_port () -{ - auto test_env = nano::get_env_or_default ("NANO_TEST_WEBSOCKET_PORT", "17078"); - return boost::lexical_cast (test_env); -} - -std::array test_magic_number () -{ - auto test_env = get_env_or_default ("NANO_TEST_MAGIC_NUMBER", "RX"); - std::array ret; - std::copy (test_env.begin (), test_env.end (), ret.data ()); - return ret; -} - void force_nano_dev_network () { nano::network_constants::set_active_network (nano::networks::nano_dev_network); @@ -316,36 +287,44 @@ std::string get_tls_toml_config_path (std::filesystem::path const & data_path) { return (data_path / "config-tls.toml").string (); } -} // namespace nano +} + +uint16_t nano::test_node_port () +{ + auto test_env = nano::env::get ("NANO_TEST_NODE_PORT").value_or ("17075"); + return boost::lexical_cast (test_env); +} -std::string nano::get_env_or_default (char const * variable_name, std::string default_value) +uint16_t nano::test_rpc_port () { - auto value = nano::env::get (variable_name); - return value.value_or (default_value); + auto test_env = nano::env::get ("NANO_TEST_RPC_PORT").value_or ("17076"); + return boost::lexical_cast (test_env); } -int nano::get_env_int_or_default (const char * variable_name, const int default_value) +uint16_t nano::test_ipc_port () { - auto value = nano::env::get (variable_name); - if (value) - { - try - { - return boost::lexical_cast (*value); - } - catch (...) - { - // It is unexpected that this exception will be caught, log to cerr the reason. - std::cerr << boost::str (boost::format ("Error parsing environment variable: %1% value: %2%") % variable_name % *value); - throw; - } - } - return default_value; + auto test_env = nano::env::get ("NANO_TEST_IPC_PORT").value_or ("17077"); + return boost::lexical_cast (test_env); +} + +uint16_t nano::test_websocket_port () +{ + auto test_env = nano::env::get ("NANO_TEST_WEBSOCKET_PORT").value_or ("17078"); + return boost::lexical_cast (test_env); +} + +std::array nano::test_magic_number () +{ + auto test_env = nano::env::get ("NANO_TEST_MAGIC_NUMBER").value_or ("RX"); + release_assert (test_env.size () == 2); + std::array ret{}; + std::copy (test_env.begin (), test_env.end (), ret.data ()); + return ret; } uint32_t nano::test_scan_wallet_reps_delay () { - auto test_env = nano::get_env_or_default ("NANO_TEST_WALLET_SCAN_REPS_DELAY", "900000"); // 15 minutes by default + auto test_env = nano::env::get ("NANO_TEST_WALLET_SCAN_REPS_DELAY").value_or ("900000"); // 15 minutes default return boost::lexical_cast (test_env); } diff --git a/nano/lib/config.hpp b/nano/lib/config.hpp index f012fc551e..2ce21f9ed8 100644 --- a/nano/lib/config.hpp +++ b/nano/lib/config.hpp @@ -89,14 +89,6 @@ uint8_t get_pre_release_node_version (); * Environment variables */ -/* - * Get environment variable as string or `default_value` if variable is not present - */ -std::string get_env_or_default (char const * variable_name, std::string const default_value); -/* - * Get environment variable as int or `default_value` if variable is not present - */ -int get_env_int_or_default (char const * variable_name, int const default_value); uint64_t get_env_threshold_or_default (char const * variable_name, uint64_t const default_value); uint16_t test_node_port (); diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index f434f5f8fa..ae86068311 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -16,9 +17,9 @@ namespace char const * preconfigured_peers_key = "preconfigured_peers"; char const * signature_checker_threads_key = "signature_checker_threads"; char const * pow_sleep_interval_key = "pow_sleep_interval"; -std::string const default_live_peer_network = nano::get_env_or_default ("NANO_DEFAULT_PEER", "peering.nano.org"); -std::string const default_beta_peer_network = nano::get_env_or_default ("NANO_DEFAULT_PEER", "peering-beta.nano.org"); -std::string const default_test_peer_network = nano::get_env_or_default ("NANO_DEFAULT_PEER", "peering-test.nano.org"); +std::string const default_live_peer_network = nano::env::get ("NANO_DEFAULT_PEER").value_or ("peering.nano.org"); +std::string const default_beta_peer_network = nano::env::get ("NANO_DEFAULT_PEER").value_or ("peering-beta.nano.org"); +std::string const default_test_peer_network = nano::env::get ("NANO_DEFAULT_PEER").value_or ("peering-test.nano.org"); } nano::node_config::node_config (nano::network_params & network_params) : diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index 3c2ad8a9a3..f5fd8e44fc 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -26,7 +27,8 @@ char const * dev_private_key_data = "34F0A37AAD20F4A260F0A5B3CB3D7FB50673212263E char const * dev_public_key_data = "B0311EA55708D6A53C75CDBF88300259C6D018522FE3D4D0A242E431F9E8B6D0"; // xrb_3e3j5tkog48pnny9dmfzj1r16pg8t1e76dz5tmac6iq689wyjfpiij4txtdo char const * beta_public_key_data = "259A438A8F9F9226130C84D902C237AF3E57C0981C7D709C288046B110D8C8AC"; // nano_1betagoxpxwykx4kw86dnhosc8t3s7ix8eeentwkcg1hbpez1outjrcyg4n1 char const * live_public_key_data = "E89208DD038FBB269987689621D52292AE9C35941A7484756ECCED92A65093BA"; // xrb_3t6k35gi95xu6tergt6p69ck76ogmitsa8mnijtpxm9fkcm736xtoncuohr3 -std::string const test_public_key_data = nano::get_env_or_default ("NANO_TEST_GENESIS_PUB", "45C6FF9D1706D61F0821327752671BDA9F9ED2DA40326B01935AB566FB9E08ED"); // nano_1jg8zygjg3pp5w644emqcbmjqpnzmubfni3kfe1s8pooeuxsw49fdq1mco9j +std::string const test_public_key_data = nano::env::get ("NANO_TEST_GENESIS_PUB").value_or ("45C6FF9D1706D61F0821327752671BDA9F9ED2DA40326B01935AB566FB9E08ED"); // nano_1jg8zygjg3pp5w644emqcbmjqpnzmubfni3kfe1s8pooeuxsw49fdq1mco9j + char const * dev_genesis_data = R"%%%({ "type": "open", "source": "B0311EA55708D6A53C75CDBF88300259C6D018522FE3D4D0A242E431F9E8B6D0", @@ -54,7 +56,7 @@ char const * live_genesis_data = R"%%%({ "signature": "9F0C933C8ADE004D808EA1985FA746A7E95BA2A38F867640F53EC8F180BDFE9E2C1268DEAD7C2664F356E37ABA362BC58E46DBA03E523A7B5A19E4B6EB12BB02" })%%%"; -std::string const test_genesis_data = nano::get_env_or_default ("NANO_TEST_GENESIS_BLOCK", R"%%%({ +std::string const test_genesis_data = nano::env::get ("NANO_TEST_GENESIS_BLOCK").value_or (R"%%%({ "type": "open", "source": "45C6FF9D1706D61F0821327752671BDA9F9ED2DA40326B01935AB566FB9E08ED", "representative": "nano_1jg8zygjg3pp5w644emqcbmjqpnzmubfni3kfe1s8pooeuxsw49fdq1mco9j", From 0ba54c27ab7b8248dd7d14825a7f02690eeabdcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 10 May 2024 12:30:33 +0200 Subject: [PATCH 06/13] Use generic get --- nano/lib/env.cpp | 35 ++--------------------------------- nano/lib/env.hpp | 41 +++++++++++++++++++++++++++++++---------- nano/lib/stats.cpp | 2 +- nano/lib/threading.cpp | 2 +- 4 files changed, 35 insertions(+), 45 deletions(-) diff --git a/nano/lib/env.cpp b/nano/lib/env.cpp index 3b67347c1d..e79d2d8ef4 100644 --- a/nano/lib/env.cpp +++ b/nano/lib/env.cpp @@ -14,7 +14,8 @@ std::optional nano::env::get (std::string_view name) return std::nullopt; } -std::optional nano::env::get_bool (std::string_view name) +template <> +std::optional nano::env::get (std::string_view name) { std::vector const on_values{ "1", "true", "on" }; std::vector const off_values{ "0", "false", "off" }; @@ -34,36 +35,4 @@ std::optional nano::env::get_bool (std::string_view name) throw std::invalid_argument ("Invalid environment boolean value: " + *value); } return std::nullopt; -} - -std::optional nano::env::get_int (std::string_view name) -{ - if (auto value = get (name)) - { - try - { - return std::stoi (*value); - } - catch (std::invalid_argument const &) - { - throw std::invalid_argument ("Invalid environment integer value: " + *value); - } - } - return std::nullopt; -} - -std::optional nano::env::get_uint (std::string_view name) -{ - if (auto value = get (name)) - { - try - { - return std::stoul (*value); - } - catch (std::invalid_argument const &) - { - throw std::invalid_argument ("Invalid environment unsigned integer value: " + *value); - } - } - return std::nullopt; } \ No newline at end of file diff --git a/nano/lib/env.hpp b/nano/lib/env.hpp index ccd911c21f..ae06440db0 100644 --- a/nano/lib/env.hpp +++ b/nano/lib/env.hpp @@ -1,21 +1,42 @@ #pragma once +#include + #include #include -/* - * Get environment variable as a specific type or none if variable is not present. - */ namespace nano::env { +/** + * Get environment variable as a specific type or none if variable is not present. + */ std::optional get (std::string_view name); -// @throws std::invalid_argument if the value is not a valid boolean -std::optional get_bool (std::string_view name); - -// @throws std::invalid_argument if the value is not a valid integer -std::optional get_int (std::string_view name); +/** + * Get environment variable as a specific type or none if variable is not present. + * @throws std::invalid_argument if the value cannot be converted + */ +template +std::optional get (std::string_view name) +{ + if (auto value = get (name)) + { + try + { + return boost::lexical_cast (*value); + } + catch (boost::bad_lexical_cast const &) + { + throw std::invalid_argument ("Invalid environment value: " + *value); + } + } + return std::nullopt; +} -// @throws std::invalid_argument if the value is not a valid integer -std::optional get_uint (std::string_view name); +/** + * Specialization for boolean values. + * @throws std::invalid_argument if the value is not a valid boolean + */ +template <> +std::optional get (std::string_view name); } \ No newline at end of file diff --git a/nano/lib/stats.cpp b/nano/lib/stats.cpp index 310648cf48..d4dfbfefb2 100644 --- a/nano/lib/stats.cpp +++ b/nano/lib/stats.cpp @@ -33,7 +33,7 @@ std::string nano::stat_log_sink::tm_to_string (tm & tm) nano::stats::stats (nano::logger & logger_a, nano::stats_config config_a) : config{ std::move (config_a) }, logger{ logger_a }, - enable_logging{ nano::env::get_bool ("NANO_LOG_STATS").value_or (false) } + enable_logging{ nano::env::get ("NANO_LOG_STATS").value_or (false) } { } diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index 5fb6b5d3a7..932816ae52 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -19,7 +19,7 @@ boost::thread::attributes nano::thread_attributes::get_default () unsigned nano::hardware_concurrency () { static auto const concurrency = [] () { - if (auto value = nano::env::get_uint ("NANO_HARDWARE_CONCURRENCY")) + if (auto value = nano::env::get ("NANO_HARDWARE_CONCURRENCY")) { std::cerr << "Hardware concurrency overridden by NANO_HARDWARE_CONCURRENCY environment variable: " << *value << std::endl; return *value; From e23e23b4ed7fc9a3909a3ec5b73c2bededf71524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 10 May 2024 12:35:29 +0200 Subject: [PATCH 07/13] Inform about environment overrides --- nano/lib/config.cpp | 78 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/nano/lib/config.cpp b/nano/lib/config.cpp index 70b136357f..bc7178c95c 100644 --- a/nano/lib/config.cpp +++ b/nano/lib/config.cpp @@ -291,41 +291,85 @@ std::string get_tls_toml_config_path (std::filesystem::path const & data_path) uint16_t nano::test_node_port () { - auto test_env = nano::env::get ("NANO_TEST_NODE_PORT").value_or ("17075"); - return boost::lexical_cast (test_env); + static auto const test_env = [] () -> std::optional { + if (auto value = nano::env::get ("NANO_TEST_NODE_PORT")) + { + std::cerr << "Node port overridden by NANO_TEST_NODE_PORT environment variable: " << *value << std::endl; + return *value; + } + return std::nullopt; + }(); + return test_env.value_or (17075); } uint16_t nano::test_rpc_port () { - auto test_env = nano::env::get ("NANO_TEST_RPC_PORT").value_or ("17076"); - return boost::lexical_cast (test_env); + static auto const test_env = [] () -> std::optional { + if (auto value = nano::env::get ("NANO_TEST_RPC_PORT")) + { + std::cerr << "RPC port overridden by NANO_TEST_RPC_PORT environment variable: " << *value << std::endl; + return *value; + } + return std::nullopt; + }(); + return test_env.value_or (17076); } uint16_t nano::test_ipc_port () { - auto test_env = nano::env::get ("NANO_TEST_IPC_PORT").value_or ("17077"); - return boost::lexical_cast (test_env); + static auto const test_env = [] () -> std::optional { + if (auto value = nano::env::get ("NANO_TEST_IPC_PORT")) + { + std::cerr << "IPC port overridden by NANO_TEST_IPC_PORT environment variable: " << *value << std::endl; + return *value; + } + return std::nullopt; + }(); + return test_env.value_or (17077); } uint16_t nano::test_websocket_port () { - auto test_env = nano::env::get ("NANO_TEST_WEBSOCKET_PORT").value_or ("17078"); - return boost::lexical_cast (test_env); + static auto const test_env = [] () -> std::optional { + if (auto value = nano::env::get ("NANO_TEST_WEBSOCKET_PORT")) + { + std::cerr << "Websocket port overridden by NANO_TEST_WEBSOCKET_PORT environment variable: " << *value << std::endl; + return *value; + } + return std::nullopt; + }(); + return test_env.value_or (17078); } -std::array nano::test_magic_number () +uint32_t nano::test_scan_wallet_reps_delay () { - auto test_env = nano::env::get ("NANO_TEST_MAGIC_NUMBER").value_or ("RX"); - release_assert (test_env.size () == 2); - std::array ret{}; - std::copy (test_env.begin (), test_env.end (), ret.data ()); - return ret; + static auto const test_env = [] () -> std::optional { + if (auto value = nano::env::get ("NANO_TEST_WALLET_SCAN_REPS_DELAY")) + { + std::cerr << "Wallet scan interval overridden by NANO_TEST_WALLET_SCAN_REPS_DELAY environment variable: " << *value << std::endl; + return *value; + } + return std::nullopt; + }(); + return test_env.value_or (900000); // 15 minutes default } -uint32_t nano::test_scan_wallet_reps_delay () +std::array nano::test_magic_number () { - auto test_env = nano::env::get ("NANO_TEST_WALLET_SCAN_REPS_DELAY").value_or ("900000"); // 15 minutes default - return boost::lexical_cast (test_env); + static auto const test_env = [] () -> std::optional { + if (auto value = nano::env::get ("NANO_TEST_MAGIC_NUMBER")) + { + std::cerr << "Magic number overridden by NANO_TEST_MAGIC_NUMBER environment variable: " << *value << std::endl; + return *value; + } + return std::nullopt; + }(); + + auto value = test_env.value_or ("RX"); + release_assert (value.size () == 2); + std::array ret{}; + std::copy (value.begin (), value.end (), ret.data ()); + return ret; } std::string_view nano::to_string (nano::networks network) From 1136a59974d2a8c2a5791ae6218685e11a9e122a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 10 May 2024 12:55:04 +0200 Subject: [PATCH 08/13] Consteval sanitizer info --- nano/lib/config.cpp | 5 ----- nano/lib/config.hpp | 45 ++++++++++++++++++--------------------------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/nano/lib/config.cpp b/nano/lib/config.cpp index bc7178c95c..f0a47407d4 100644 --- a/nano/lib/config.cpp +++ b/nano/lib/config.cpp @@ -258,11 +258,6 @@ bool slow_instrumentation () return is_tsan_build () || nano::running_within_valgrind (); } -bool is_sanitizer_build () -{ - return is_asan_build () || is_tsan_build (); -} - std::string get_node_toml_config_path (std::filesystem::path const & data_path) { return (data_path / "config-node.toml").string (); diff --git a/nano/lib/config.hpp b/nano/lib/config.hpp index 2ce21f9ed8..5de34c0925 100644 --- a/nano/lib/config.hpp +++ b/nano/lib/config.hpp @@ -28,55 +28,49 @@ char const * const NANO_PRE_RELEASE_VERSION_STRING = xstr (PRE_RELEASE_VERSION_S char const * const BUILD_INFO = xstr (GIT_COMMIT_HASH BOOST_COMPILER) " \"BOOST " xstr (BOOST_VERSION) "\" BUILT " xstr (__DATE__); +/* + * Sanitizer info + */ +namespace nano +{ +consteval bool is_asan_build () +{ #if defined(__has_feature) #if __has_feature(address_sanitizer) -inline bool is_asan_build () -{ return true; -} #else -inline bool is_asan_build () -{ return false; -} #endif // GCC builds #elif defined(__SANITIZE_ADDRESS__) -inline bool is_asan_build () -{ return true; -} #else -inline bool is_asan_build () -{ return false; -} #endif +} +consteval bool is_tsan_build () +{ #if defined(__has_feature) #if __has_feature(thread_sanitizer) -inline bool is_tsan_build () -{ return true; -} #else -inline bool is_tsan_build () -{ return false; -} #endif // GCC builds #elif defined(__SANITIZE_THREAD__) -inline bool is_tsan_build () -{ return true; -} #else -inline bool is_tsan_build () -{ return false; -} #endif +} + +/** Checks if we are running with either AddressSanitizer or ThreadSanitizer */ +consteval bool is_sanitizer_build () +{ + return is_asan_build () || is_tsan_build (); +} +} namespace nano { @@ -404,9 +398,6 @@ bool memory_intensive_instrumentation (); Returns true if running within Valgrind or with ThreadSanitizer tooling*/ bool slow_instrumentation (); -/** Checks if we are running with either AddressSanitizer or ThreadSanitizer*/ -bool is_sanitizer_build (); - /** Set the active network to the dev network */ void force_nano_dev_network (); From fb4b150f69c0115030202daaaa4b4c1461068e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 10 May 2024 17:20:46 +0200 Subject: [PATCH 09/13] Print stats logging info --- nano/lib/stats.cpp | 15 ++++++++++++++- nano/lib/stats.hpp | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/nano/lib/stats.cpp b/nano/lib/stats.cpp index d4dfbfefb2..55511e9d92 100644 --- a/nano/lib/stats.cpp +++ b/nano/lib/stats.cpp @@ -33,7 +33,7 @@ std::string nano::stat_log_sink::tm_to_string (tm & tm) nano::stats::stats (nano::logger & logger_a, nano::stats_config config_a) : config{ std::move (config_a) }, logger{ logger_a }, - enable_logging{ nano::env::get ("NANO_LOG_STATS").value_or (false) } + enable_logging{ is_stat_logging_enabled () } { } @@ -365,6 +365,19 @@ std::string nano::stats::dump (category category) return sink.to_string (); } +bool nano::stats::is_stat_logging_enabled () +{ + static auto const enabled = [] () { + if (auto value = nano::env::get ("NANO_LOG_STATS")) + { + std::cerr << "Stats logging enabled by NANO_LOG_STATS environment variable" << std::endl; + return *value; + } + return false; + }(); + return enabled; +} + /* * stats::sampler_entry */ diff --git a/nano/lib/stats.hpp b/nano/lib/stats.hpp index 9bb1006715..905153f957 100644 --- a/nano/lib/stats.hpp +++ b/nano/lib/stats.hpp @@ -199,6 +199,8 @@ class stats final /** Unlocked implementation of log_samples() to avoid using recursive locking */ void log_samples_impl (stat_log_sink & sink, tm & tm); + static bool is_stat_logging_enabled (); + private: nano::stats_config const config; nano::logger & logger; From 694dec236a534158b0ba7ea3ff81838830558d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 10 May 2024 23:11:06 +0200 Subject: [PATCH 10/13] Replace `get_env_threshold_or_default` --- nano/lib/config.cpp | 21 ++++++++++++--------- nano/lib/config.hpp | 10 +--------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/nano/lib/config.cpp b/nano/lib/config.cpp index f0a47407d4..0674e605f7 100644 --- a/nano/lib/config.cpp +++ b/nano/lib/config.cpp @@ -16,10 +16,19 @@ template struct HexTo { ElemT value; + + HexTo () = default; + + HexTo (ElemT val) : + value{ val } + { + } + operator ElemT () const { return value; } + friend std::istream & operator>> (std::istream & in, HexTo & out) { in >> std::hex >> out.value; @@ -47,9 +56,9 @@ nano::work_thresholds const nano::work_thresholds::publish_dev ( ); nano::work_thresholds const nano::work_thresholds::publish_test ( // defaults to live network levels -get_env_threshold_or_default ("NANO_TEST_EPOCH_1", 0xffffffc000000000), -get_env_threshold_or_default ("NANO_TEST_EPOCH_2", 0xfffffff800000000), // 8x higher than epoch_1 -get_env_threshold_or_default ("NANO_TEST_EPOCH_2_RECV", 0xfffffe0000000000) // 8x lower than epoch_1 +nano::env::get> ("NANO_TEST_EPOCH_1").value_or (0xffffffc000000000), +nano::env::get> ("NANO_TEST_EPOCH_2").value_or (0xfffffff800000000), // 8x higher than epoch_1 +nano::env::get> ("NANO_TEST_EPOCH_2_RECV").value_or (0xfffffe0000000000) // 8x lower than epoch_1 ); uint64_t nano::work_thresholds::threshold_entry (nano::work_version const version_a, nano::block_type const type_a) const @@ -232,12 +241,6 @@ uint8_t get_pre_release_node_version () return boost::numeric_cast (boost::lexical_cast (NANO_PRE_RELEASE_VERSION_STRING)); } -uint64_t get_env_threshold_or_default (char const * variable_name, uint64_t const default_value) -{ - auto * value = getenv (variable_name); - return value ? boost::lexical_cast> (value) : default_value; -} - void force_nano_dev_network () { nano::network_constants::set_active_network (nano::networks::nano_dev_network); diff --git a/nano/lib/config.hpp b/nano/lib/config.hpp index 5de34c0925..2766af1e8f 100644 --- a/nano/lib/config.hpp +++ b/nano/lib/config.hpp @@ -79,20 +79,12 @@ uint8_t get_minor_node_version (); uint8_t get_patch_node_version (); uint8_t get_pre_release_node_version (); -/* - * Environment variables - */ - -uint64_t get_env_threshold_or_default (char const * variable_name, uint64_t const default_value); - uint16_t test_node_port (); uint16_t test_rpc_port (); uint16_t test_ipc_port (); uint16_t test_websocket_port (); std::array test_magic_number (); -/* - * How often to scan for representatives in local wallet, in milliseconds - */ +/// How often to scan for representatives in local wallet, in milliseconds uint32_t test_scan_wallet_reps_delay (); /** From c523877caf1840240158124fa76c04863b2f7e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sat, 11 May 2024 10:21:55 +0200 Subject: [PATCH 11/13] Allow overriding io threads from env variable --- nano/node/nodeconfig.cpp | 13 +++++++++++++ nano/node/nodeconfig.hpp | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index ae86068311..faf9838b98 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -657,3 +657,16 @@ nano::account nano::node_config::random_representative () const auto result (preconfigured_representatives[index]); return result; } + +std::optional nano::node_config::env_io_threads () +{ + static auto const value = [] () { + auto value = nano::env::get ("NANO_IO_THREADS"); + if (value) + { + std::cerr << "IO threads overridden by NANO_IO_THREADS environment variable: " << *value << std::endl; + } + return value; + }(); + return value; +} diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index bdd16bfd56..ff9f31d87d 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -82,7 +82,7 @@ class node_config */ nano::amount representative_vote_weight_minimum{ 10 * nano::Mxrb_ratio }; unsigned password_fanout{ 1024 }; - unsigned io_threads{ std::max (4u, nano::hardware_concurrency ()) }; + unsigned io_threads{ env_io_threads ().value_or (std::max (4u, nano::hardware_concurrency ())) }; unsigned network_threads{ std::max (4u, nano::hardware_concurrency ()) }; unsigned work_threads{ std::max (4u, nano::hardware_concurrency ()) }; unsigned background_threads{ std::max (4u, nano::hardware_concurrency ()) }; @@ -154,6 +154,9 @@ class node_config nano::frontiers_confirmation_mode deserialize_frontiers_confirmation (std::string const &); /** Entry is ignored if it cannot be parsed as a valid address:port */ void deserialize_address (std::string const &, std::vector> &) const; + +private: + static std::optional env_io_threads (); }; class node_flags final From 3fc705b2865962d404e8348efab8bdba7baa00b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sat, 11 May 2024 10:34:11 +0200 Subject: [PATCH 12/13] Ensure configured thread counts are in reasonable ranges --- nano/lib/threading.cpp | 1 + nano/lib/utility.hpp | 10 ++++++++++ nano/node/message_processor.hpp | 2 +- nano/node/request_aggregator.hpp | 2 +- nano/node/vote_processor.hpp | 2 +- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index 932816ae52..504c2628d8 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -26,6 +26,7 @@ unsigned nano::hardware_concurrency () } return std::thread::hardware_concurrency (); }(); + release_assert (concurrency > 0, "configured hardware concurrency must be non zero"); return concurrency; } diff --git a/nano/lib/utility.hpp b/nano/lib/utility.hpp index 35d19892dd..1ec1998e6d 100644 --- a/nano/lib/utility.hpp +++ b/nano/lib/utility.hpp @@ -285,4 +285,14 @@ std::string to_str (T const & val) { return boost::lexical_cast (val); } +} + +namespace nano +{ +template +T min_max (T min, T max, T value) +{ + debug_assert (min <= max); + return std::min (max, std::max (min, value)); +} } \ No newline at end of file diff --git a/nano/node/message_processor.hpp b/nano/node/message_processor.hpp index e160b3396a..9ed741f72e 100644 --- a/nano/node/message_processor.hpp +++ b/nano/node/message_processor.hpp @@ -17,7 +17,7 @@ class message_processor_config final nano::error serialize (nano::tomlconfig & toml) const; public: - size_t threads{ std::min (nano::hardware_concurrency () / 4, 2u) }; + size_t threads{ min_max (1u, 2u, nano::hardware_concurrency () / 4) }; size_t max_queue{ 64 }; }; diff --git a/nano/node/request_aggregator.hpp b/nano/node/request_aggregator.hpp index ebe73fb3ef..bcc9a064eb 100644 --- a/nano/node/request_aggregator.hpp +++ b/nano/node/request_aggregator.hpp @@ -29,7 +29,7 @@ class request_aggregator_config final nano::error serialize (nano::tomlconfig &) const; public: - size_t threads{ std::min (nano::hardware_concurrency (), 4u) }; + size_t threads{ min_max (1u, 4u, nano::hardware_concurrency () / 2) }; size_t max_queue{ 128 }; size_t batch_size{ 16 }; }; diff --git a/nano/node/vote_processor.hpp b/nano/node/vote_processor.hpp index f9b8acbcd7..6d699dea08 100644 --- a/nano/node/vote_processor.hpp +++ b/nano/node/vote_processor.hpp @@ -50,7 +50,7 @@ class vote_processor_config final size_t max_pr_queue{ 256 }; size_t max_non_pr_queue{ 32 }; size_t pr_priority{ 3 }; - size_t threads{ std::min (4u, nano::hardware_concurrency () / 2) }; + size_t threads{ min_max (1u, 4u, nano::hardware_concurrency () / 2) }; size_t batch_size{ 1024 }; }; From 978683534b850ca4974ac39e5de6983faf98d64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 13 May 2024 10:53:30 +0200 Subject: [PATCH 13/13] Use `std::clamp` --- nano/lib/utility.hpp | 10 ---------- nano/node/message_processor.hpp | 2 +- nano/node/request_aggregator.hpp | 2 +- nano/node/vote_processor.hpp | 2 +- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/nano/lib/utility.hpp b/nano/lib/utility.hpp index 1ec1998e6d..35d19892dd 100644 --- a/nano/lib/utility.hpp +++ b/nano/lib/utility.hpp @@ -285,14 +285,4 @@ std::string to_str (T const & val) { return boost::lexical_cast (val); } -} - -namespace nano -{ -template -T min_max (T min, T max, T value) -{ - debug_assert (min <= max); - return std::min (max, std::max (min, value)); -} } \ No newline at end of file diff --git a/nano/node/message_processor.hpp b/nano/node/message_processor.hpp index 9ed741f72e..5481dc6887 100644 --- a/nano/node/message_processor.hpp +++ b/nano/node/message_processor.hpp @@ -17,7 +17,7 @@ class message_processor_config final nano::error serialize (nano::tomlconfig & toml) const; public: - size_t threads{ min_max (1u, 2u, nano::hardware_concurrency () / 4) }; + size_t threads{ std::clamp (nano::hardware_concurrency () / 4, 1u, 2u) }; size_t max_queue{ 64 }; }; diff --git a/nano/node/request_aggregator.hpp b/nano/node/request_aggregator.hpp index bcc9a064eb..eb3d7fd934 100644 --- a/nano/node/request_aggregator.hpp +++ b/nano/node/request_aggregator.hpp @@ -29,7 +29,7 @@ class request_aggregator_config final nano::error serialize (nano::tomlconfig &) const; public: - size_t threads{ min_max (1u, 4u, nano::hardware_concurrency () / 2) }; + size_t threads{ std::clamp (nano::hardware_concurrency () / 2, 1u, 4u) }; size_t max_queue{ 128 }; size_t batch_size{ 16 }; }; diff --git a/nano/node/vote_processor.hpp b/nano/node/vote_processor.hpp index 6d699dea08..7c56f4ac1c 100644 --- a/nano/node/vote_processor.hpp +++ b/nano/node/vote_processor.hpp @@ -50,7 +50,7 @@ class vote_processor_config final size_t max_pr_queue{ 256 }; size_t max_non_pr_queue{ 32 }; size_t pr_priority{ 3 }; - size_t threads{ min_max (1u, 4u, nano::hardware_concurrency () / 2) }; + size_t threads{ std::clamp (nano::hardware_concurrency () / 2, 1u, 4u) }; size_t batch_size{ 1024 }; };