Skip to content

Commit

Permalink
migrate_database_lmdb_to_rocksdb improvements (#4647)
Browse files Browse the repository at this point in the history 
* Rocksdb migration: disk space check and progress output

* Estimate the required free disk space

* More granular progress feedback

* Use Nano Logger

* Clang format fix

* Fixed double count

* Use atomics for multithreaded counters are accurate

* Use a single periodically refreshed transaction per thread.

* Output the number of entries for each table

* Use TransactionDB rather than OptimisticTransactionDB.

* Use default refresh interval.

* Fixed error and aligned log output format

---------

Co-authored-by: Colin LeMahieu <clemahieu@gmail.com>
  • Loading branch information
@RickiNano @clemahieu
RickiNano and clemahieu authored Jul 10, 2024
1 parent 4f450ae commit 809becf
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 22 deletions.
1 change: 1 addition & 0 deletions nano/lib/logging_enums.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ enum class type
opencl_work,
upnp,
rep_crawler,
ledger,
lmdb,
rocksdb,
txn_tracker,
Expand Down
7 changes: 1 addition & 6 deletions nano/node/cli.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,19 +483,14 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
auto error (false);
if (!node.node->init_error ())
{
std::cout << "Migrating LMDB database to RocksDB, might take a while..." << std::endl;
error = node.node->ledger.migrate_lmdb_to_rocksdb (data_path);
}
else
{
error = true;
}

if (!error)
{
std::cout << "Migration completed, after confirming it is correct the data.ldb file can be deleted if no longer required" << std::endl;
}
else
if (error)
{
std::cerr << "There was an error migrating" << std::endl;
}
Expand Down
112 changes: 96 additions & 16 deletions nano/secure/ledger.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1241,12 +1241,25 @@ uint64_t nano::ledger::pruning_action (secure::write_transaction & transaction_a
// A precondition is that the store is an LMDB store
bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_path_a) const
{
nano::logger logger;
nano::logger::initialize (nano::log_config::daemon_default (), data_path_a);

logger.info (nano::log::type::ledger, "Migrating LMDB database to RocksDB. This will take a while...");

std::filesystem::space_info si = std::filesystem::space (data_path_a);
auto file_size = std::filesystem::file_size (data_path_a / "data.ldb");
const auto estimated_required_space = file_size * 0.65; // RocksDb database size is approximately 65% of the lmdb size

if (si.available < estimated_required_space)
{
logger.warn (nano::log::type::ledger, "You may not have enough available disk space. Estimated free space requirement is {} GB", estimated_required_space / 1024 / 1024 / 1024);
}

boost::system::error_code error_chmod;
nano::set_secure_perm_directory (data_path_a, error_chmod);
auto rockdb_data_path = data_path_a / "rocksdb";
std::filesystem::remove_all (rockdb_data_path);

nano::logger logger;
auto error (false);

// Open rocksdb database
Expand All @@ -1256,76 +1269,140 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p

if (!rocksdb_store->init_error ())
{
auto table_size = store.count (store.tx_begin_read (), tables::blocks);
logger.info (nano::log::type::ledger, "Step 1 of 7: Converting {} entries from blocks table", table_size);
std::atomic<std::size_t> count = 0;
store.block.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks }));

rocksdb_transaction.refresh_if_needed ();
std::vector<uint8_t> vector;
{
nano::vectorstream stream (vector);
nano::serialize_block (stream, *i->second.block);
i->second.sideband.serialize (stream, i->second.block->type ());
}
rocksdb_store->block.raw_put (rocksdb_transaction, vector, i->first);

if (auto count_l = ++count; count_l % 5000000 == 0)
{
logger.info (nano::log::type::ledger, "{} blocks converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} blocks", count.load ());

table_size = store.count (store.tx_begin_read (), tables::pending);
logger.info (nano::log::type::ledger, "Step 2 of 7: Converting {} entries from pending table", table_size);
count = 0;
store.pending.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->pending.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::confirmation_height);
logger.info (nano::log::type::ledger, "Step 3 of 7: Converting {} entries from confirmation_height table", table_size);
count = 0;
store.confirmation_height.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->confirmation_height.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::accounts);
logger.info (nano::log::type::ledger, "Step 4 of 7: Converting {} entries from accounts table", table_size);
count = 0;
store.account.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->account.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::rep_weights);
logger.info (nano::log::type::ledger, "Step 5 of 7: Converting {} entries from rep_weights table", table_size);
count = 0;
store.rep_weight.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::rep_weights }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::rep_weights }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->rep_weight.put (rocksdb_transaction, i->first, i->second.number ());
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::pruned);
logger.info (nano::log::type::ledger, "Step 6 of 7: Converting {} entries from pruned table", table_size);
count = 0;
store.pruned.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->pruned.put (rocksdb_transaction, i->first);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::final_votes);
logger.info (nano::log::type::ledger, "Step 7 of 7: Converting {} entries from final_votes table", table_size);
count = 0;
store.final_vote.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->final_vote.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

logger.info (nano::log::type::ledger, "Finalizing migration...");
auto lmdb_transaction (store.tx_begin_read ());
auto version = store.version.get (lmdb_transaction);
auto rocksdb_transaction (rocksdb_store->tx_begin_write ());
Expand Down Expand Up @@ -1362,6 +1439,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p
{
error |= rocksdb_store->confirmation_height.get (rocksdb_transaction, account, confirmation_height_info);
}

logger.info (nano::log::type::ledger, "Migration completed. Make sure to enable RocksDb in the config file under [node.rocksdb]");
logger.info (nano::log::type::ledger, "After confirming correct node operation, the data.ldb file can be deleted if no longer required");
}
else
{
Expand Down

0 comments on commit 809becf

Please sign in to comment.