Skip to content

Commit

Permalink
fcmp++: fix migration batching to resize db as needed
Browse files Browse the repository at this point in the history
- batch_start is the simplest function to use to resize db, since
resizing requires no active txns.
- batch_stop makes sure no active txns.
- need to decrement txns before calling migrate() so that do_resize
does not deadlock in wait_no_active_txns
  • Loading branch information
j-berman committed Oct 29, 2024
1 parent 1241172 commit 3ac3e53
Showing 1 changed file with 24 additions and 24 deletions.
48 changes: 24 additions & 24 deletions src/blockchain_db/lmdb/db_lmdb.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2642,7 +2642,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
// We don't handle the old format previous to that commit.
txn.commit();
m_open = true;
// Decrement num active txs so db can resize if needed
mdb_txn_safe::increment_txns(-1);
migrate(db_version);
mdb_txn_safe::increment_txns(1);
return;
}
#endif
Expand Down Expand Up @@ -6838,7 +6841,9 @@ void BlockchainLMDB::migrate_5_6()
if (!m_batch_transactions)
set_batch_transactions(true);
const std::size_t BATCH_SIZE = 10000;
batch_start(BATCH_SIZE);
// Since step 3/3 in migration deletes block info records from the db, can't use num blocks batch size, otherwise
// get_estimated_batch_size can fail to read block weight by height
batch_start();
txn.m_txn = m_write_txn->m_txn;

/* the spent_keys table name is the same but the old version and new version
Expand Down Expand Up @@ -6883,13 +6888,10 @@ void BlockchainLMDB::migrate_5_6()
std::cout << i << " / " << n_key_images << " key images (" << percent << "% of step 1/3) \r" << std::flush;
}

txn.commit();
result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
m_write_txn->m_txn = txn.m_txn;
m_write_batch_txn->m_txn = txn.m_txn;
memset(&m_wcursors, 0, sizeof(m_wcursors));
// Start a new batch so resizing can occur as needed
batch_stop();
batch_start();
txn.m_txn = m_write_txn->m_txn;
}

// Open all cursors
Expand Down Expand Up @@ -6949,7 +6951,7 @@ void BlockchainLMDB::migrate_5_6()
if (!m_batch_transactions)
set_batch_transactions(true);
const std::size_t BATCH_SIZE = 10000;
batch_start(BATCH_SIZE);
batch_start();
txn.m_txn = m_write_txn->m_txn;

// Use this cache to know how to restart the migration if the process is killed
Expand Down Expand Up @@ -6981,13 +6983,15 @@ void BlockchainLMDB::migrate_5_6()
throw0(DB_ERROR(lmdb_error("Failed to update max output id: ", result).c_str()));

// Commit and start a new txn
txn.commit();
result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
m_write_txn->m_txn = txn.m_txn;
m_write_batch_txn->m_txn = txn.m_txn;
memset(&m_wcursors, 0, sizeof(m_wcursors));
batch_stop();
batch_start();
txn.m_txn = m_write_txn->m_txn;

// Reset k and v so we continue migration from the last output
k = {sizeof(last_output.amount), (void *)&last_output.amount};

const std::size_t outkey_size = (last_output.amount == 0) ? sizeof(outkey) : sizeof(pre_rct_outkey);
v = {outkey_size, (void *)&last_output.ok};
}

// Open all cursors
Expand Down Expand Up @@ -7127,7 +7131,7 @@ void BlockchainLMDB::migrate_5_6()
if (!m_batch_transactions)
set_batch_transactions(true);
const std::size_t BATCH_SIZE = 50;
batch_start(BATCH_SIZE);
batch_start();
txn.m_txn = m_write_txn->m_txn;

/* the block_info table name is the same but the old version and new version
Expand Down Expand Up @@ -7155,13 +7159,9 @@ void BlockchainLMDB::migrate_5_6()
std::cout << i << " / " << n_blocks << " blocks (" << percent << "% of step 3/3) \r" << std::flush;
}

txn.commit();
result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
m_write_txn->m_txn = txn.m_txn;
m_write_batch_txn->m_txn = txn.m_txn;
memset(&m_wcursors, 0, sizeof(m_wcursors));
batch_stop();
batch_start();
txn.m_txn = m_write_txn->m_txn;
}

// Open all cursors
Expand Down

0 comments on commit 3ac3e53

Please sign in to comment.