Skip to content

Commit

Permalink
Merge pull request #1659 from jamescowens/bdb_opt
Browse files Browse the repository at this point in the history
Optimize bdb to avoid synchronous flush of database
  • Loading branch information
jamescowens authored Apr 10, 2020
2 parents 86c192e + 12c3f38 commit 9a56c82
Show file tree
Hide file tree
Showing 7 changed files with 68 additions and 41 deletions.
37 changes: 24 additions & 13 deletions src/db.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -233,14 +233,16 @@ void CDBEnv::lsn_reset(const std::string& strFile)
dbenv.lsn_reset(strFile.c_str(),0);
}

CDB::CDB(const char *pszFile, const char* pszMode) :
CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) :
pdb(NULL), activeTxn(NULL)
{
int ret;
if (pszFile == NULL)
return;

if (strFilename.empty())
return;

fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
fFlushOnClose = fFlushOnCloseIn;
bool fCreate = strchr(pszMode, 'c');
unsigned int nFlags = DB_THREAD;
if (fCreate)
Expand All @@ -251,7 +253,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) :
if (!bitdb.Open(GetDataDir()))
throw runtime_error("env open failed");

strFile = pszFile;
strFile = strFilename;
++bitdb.mapFileUseCount[strFile];
pdb = bitdb.mapDb[strFile];
if (pdb == NULL)
Expand All @@ -264,11 +266,11 @@ CDB::CDB(const char *pszFile, const char* pszMode) :
DbMpoolFile*mpf = pdb->get_mpf();
ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
if (ret != 0)
throw runtime_error(strprintf("CDB() : failed to configure for no temp file backing for database %s", pszFile));
throw runtime_error(strprintf("CDB() : failed to configure for no temp file backing for database %s", strFile));
}

ret = pdb->open(NULL, // Txn pointer
fMockDb ? NULL : pszFile, // Filename
fMockDb ? NULL : strFile.c_str(), // Filename
"main", // Logical db name
DB_BTREE, // Database type
nFlags, // Flags
Expand All @@ -280,7 +282,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) :
pdb = NULL;
--bitdb.mapFileUseCount[strFile];
strFile = "";
throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret));
throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", strFile, ret));
}

if (fCreate && !Exists(string("version")))
Expand All @@ -296,6 +298,19 @@ CDB::CDB(const char *pszFile, const char* pszMode) :
}
}

void CDB::Flush()
{
if (activeTxn)
return;

// Flush database activity from memory pool to disk log
unsigned int nMinutes = 0;
if (fReadOnly)
nMinutes = 1;

bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100) * 1024 : 0, nMinutes, 0);
}

void CDB::Close()
{
if (!pdb)
Expand All @@ -305,12 +320,8 @@ void CDB::Close()
activeTxn = NULL;
pdb = NULL;

// Flush database activity from memory pool to disk log
unsigned int nMinutes = 0;
if (fReadOnly)
nMinutes = 1;

bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0);
if (fFlushOnClose)
Flush();

{
LOCK(bitdb.cs_db);
Expand Down
4 changes: 3 additions & 1 deletion src/db.h
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,12 @@ class CDB
std::string strFile;
DbTxn *activeTxn;
bool fReadOnly;
bool fFlushOnClose;

explicit CDB(const char* pszFile, const char* pszMode="r+");
explicit CDB(const std::string &strFilename, const char* pszMode="r+", bool fFlushOnCloseIn = true);
~CDB() { Close(); }
public:
void Flush();
void Close();
private:
CDB(const CDB&);
Expand Down
6 changes: 3 additions & 3 deletions src/test/accounting_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade)
walletdb.WriteAccountingEntry(ae);

wtx.mapValue["comment"] = "z";
pwalletMain->AddToWallet(wtx);
pwalletMain->AddToWallet(wtx, &walletdb);
vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]);
vpwtx[0]->nTimeReceived = (unsigned int)1333333335;
vpwtx[0]->nOrderPos = -1;
Expand Down Expand Up @@ -78,13 +78,13 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade)

wtx.mapValue["comment"] = "y";
--wtx.nLockTime; // Just to change the hash :)
pwalletMain->AddToWallet(wtx);
pwalletMain->AddToWallet(wtx, &walletdb);
vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]);
vpwtx[1]->nTimeReceived = (unsigned int)1333333336;

wtx.mapValue["comment"] = "x";
--wtx.nLockTime; // Just to change the hash :)
pwalletMain->AddToWallet(wtx);
pwalletMain->AddToWallet(wtx, &walletdb);
vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]);
vpwtx[2]->nTimeReceived = (unsigned int)1333333329;
vpwtx[2]->nOrderPos = -1;
Expand Down
50 changes: 32 additions & 18 deletions src/wallet.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ CWallet::TxItems CWallet::OrderedTxItems(std::list<CAccountingEntry>& acentries,
return txOrdered;
}

void CWallet::WalletUpdateSpent(const CTransaction &tx, bool fBlock)
void CWallet::WalletUpdateSpent(const CTransaction &tx, bool fBlock, CWalletDB* pwalletdb)
{
// Anytime a signature is successfully verified, it's proof the outpoint is spent.
// Update the wallet spent flag if it doesn't know due to wallet.dat being
Expand All @@ -402,7 +402,7 @@ void CWallet::WalletUpdateSpent(const CTransaction &tx, bool fBlock)
{
if (fDebug) LogPrintf("WalletUpdateSpent found spent coin %s gC %s", FormatMoney(wtx.GetCredit()), wtx.GetHash().ToString());
wtx.MarkSpent(txin.prevout.n);
wtx.WriteToDisk();
wtx.WriteToDisk(pwalletdb);
NotifyTransactionChanged(this, txin.prevout.hash, CT_UPDATED);
}
}
Expand All @@ -419,7 +419,7 @@ void CWallet::WalletUpdateSpent(const CTransaction &tx, bool fBlock)
if (IsMine(txout))
{
wtx.MarkUnspent(&txout - &tx.vout[0]);
wtx.WriteToDisk();
wtx.WriteToDisk(pwalletdb);
NotifyTransactionChanged(this, hash, CT_UPDATED);
}
}
Expand All @@ -437,7 +437,7 @@ void CWallet::MarkDirty()
}
}

bool CWallet::AddToWallet(const CWalletTx& wtxIn)
bool CWallet::AddToWallet(const CWalletTx& wtxIn, CWalletDB* pwalletdb)
{
uint256 hash = wtxIn.GetHash();
{
Expand All @@ -450,7 +450,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn)
if (fInsertedNew)
{
wtx.nTimeReceived = GetAdjustedTime();
wtx.nOrderPos = IncOrderPosNext();
wtx.nOrderPos = IncOrderPosNext(pwalletdb);

wtx.nTimeSmart = wtx.nTimeReceived;
if (!wtxIn.hashBlock.IsNull())
Expand Down Expand Up @@ -527,7 +527,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn)

// Write to disk
if (fInsertedNew || fUpdated)
if (!wtx.WriteToDisk())
if (!wtx.WriteToDisk(pwalletdb))
return false;
if(!fQtActive)
{
Expand All @@ -550,7 +550,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn)
}
}
// since AddToWallet is called directly for self-originating transactions, check for consumption of own coins
WalletUpdateSpent(wtx, (!wtxIn.hashBlock.IsNull()));
WalletUpdateSpent(wtx, (!wtxIn.hashBlock.IsNull()), pwalletdb);

// Notify UI of new or updated transaction
NotifyTransactionChanged(this, hash, fInsertedNew ? CT_NEW : CT_UPDATED);
Expand All @@ -577,16 +577,22 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl
LOCK(cs_wallet);
bool fExisted = mapWallet.count(hash);
if (fExisted && !fUpdate) return false;

// Do not flush the wallet here for performance reasons
// this is safe, as in case of a crash, we rescan the necessary blocks on startup.
CWalletDB walletdb(strWalletFile, "r+", false);

if (fExisted || IsMine(tx) || IsFromMe(tx))
{
CWalletTx wtx(this,tx);
// Get merkle branch if transaction was found in a block
if (pblock)
wtx.SetMerkleBranch(pblock);
return AddToWallet(wtx);

return AddToWallet(wtx, &walletdb);
}
else
WalletUpdateSpent(tx);
WalletUpdateSpent(tx, false, &walletdb);
}
return false;
}
Expand Down Expand Up @@ -965,9 +971,9 @@ void CWalletTx::AddSupportingTransactions(CTxDB& txdb)
reverse(vtxPrev.begin(), vtxPrev.end());
}

bool CWalletTx::WriteToDisk()
bool CWalletTx::WriteToDisk(CWalletDB *pwalletdb)
{
return CWalletDB(pwallet->strWalletFile).WriteTx(GetHash(), *this);
return pwalletdb->WriteTx(GetHash(), *this);
}

// Scan the block chain (starting in pindexStart) for transactions
Expand Down Expand Up @@ -1042,7 +1048,10 @@ void CWallet::ReacceptWalletTransactions()
{
LogPrintf("ReacceptWalletTransactions found spent coin %s gC %s", FormatMoney(wtx.GetCredit()), wtx.GetHash().ToString());
wtx.MarkDirty();
wtx.WriteToDisk();

CWalletDB walletdb(strWalletFile);

wtx.WriteToDisk(&walletdb);
}
}
else
Expand Down Expand Up @@ -1829,14 +1838,14 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey)
// This is only to keep the database open to defeat the auto-flush for the
// duration of this scope. This is the only place where this optimization
// maybe makes sense; please don't do it anywhere else.
CWalletDB* pwalletdb = fFileBacked ? new CWalletDB(strWalletFile,"r") : NULL;
CWalletDB* pwalletdb = fFileBacked ? new CWalletDB(strWalletFile,"r+") : NULL;

// Take key pair from key pool so it won't be used again
reservekey.KeepKey();

// Add tx to wallet, because if it has change it's also ours,
// otherwise just for transaction history.
AddToWallet(wtxNew);
AddToWallet(wtxNew, pwalletdb);

// Mark old coins as spent
set<CWalletTx*> setCoins;
Expand All @@ -1845,7 +1854,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey)
CWalletTx &coin = mapWallet[txin.prevout.hash];
coin.BindWallet(this);
coin.MarkSpent(txin.prevout.n);
coin.WriteToDisk();
coin.WriteToDisk(pwalletdb);
NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED);
}

Expand Down Expand Up @@ -2380,6 +2389,8 @@ void CWallet::FixSpentCoins(int& nMismatchFound, int64_t& nBalanceInQuestion, bo
for (map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it)
vCoins.push_back(&(*it).second);

CWalletDB walletdb(strWalletFile);

CTxDB txdb("r");
for (auto const& pcoin : vCoins)
{
Expand All @@ -2398,7 +2409,7 @@ void CWallet::FixSpentCoins(int& nMismatchFound, int64_t& nBalanceInQuestion, bo
if (!fCheckOnly)
{
pcoin->MarkUnspent(n);
pcoin->WriteToDisk();
pcoin->WriteToDisk(&walletdb);
}
}
else if (IsMine(pcoin->vout[n]) && !pcoin->IsSpent(n) && (txindex.vSpent.size() > n && !txindex.vSpent[n].IsNull()))
Expand All @@ -2410,7 +2421,7 @@ void CWallet::FixSpentCoins(int& nMismatchFound, int64_t& nBalanceInQuestion, bo
if (!fCheckOnly)
{
pcoin->MarkSpent(n);
pcoin->WriteToDisk();
pcoin->WriteToDisk(&walletdb);
}
}
}
Expand All @@ -2424,6 +2435,9 @@ void CWallet::DisableTransaction(const CTransaction &tx)
return; // only disconnecting coinstake requires marking input unspent

LOCK(cs_wallet);

CWalletDB walletdb(strWalletFile);

for (auto const& txin : tx.vin)
{
map<uint256, CWalletTx>::iterator mi = mapWallet.find(txin.prevout.hash);
Expand All @@ -2433,7 +2447,7 @@ void CWallet::DisableTransaction(const CTransaction &tx)
if (txin.prevout.n < prev.vout.size() && IsMine(prev.vout[txin.prevout.n]))
{
prev.MarkUnspent(txin.prevout.n);
prev.WriteToDisk();
prev.WriteToDisk(&walletdb);
}
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/wallet.h
Original file line number Diff line number Diff line change
Expand Up @@ -247,10 +247,10 @@ class CWallet : public CCryptoKeyStore
TxItems OrderedTxItems(std::list<CAccountingEntry>& acentries, std::string strAccount = "");

void MarkDirty();
bool AddToWallet(const CWalletTx& wtxIn);
bool AddToWallet(const CWalletTx& wtxIn, CWalletDB *pwalletdb);
bool AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pblock, bool fUpdate = false, bool fFindBlock = false);
bool EraseFromWallet(uint256 hash);
void WalletUpdateSpent(const CTransaction& prevout, bool fBlock = false);
void WalletUpdateSpent(const CTransaction &tx, bool fBlock, CWalletDB* pwalletdb);
int ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false);
void ReacceptWalletTransactions();
void ResendWalletTransactions(bool fForce = false);
Expand Down Expand Up @@ -823,7 +823,7 @@ class CWalletTx : public CMerkleTx
return true;
}

bool WriteToDisk();
bool WriteToDisk(CWalletDB *pwalletdb);

int64_t GetTxTime() const;
int GetRequestCount() const;
Expand Down
4 changes: 2 additions & 2 deletions src/walletdb.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -654,7 +654,7 @@ void ThreadFlushWalletDB(void* parg)
map<string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile);
if (mi != bitdb.mapFileUseCount.end())
{
if (fDebug10) LogPrintf("Flushing wallet.dat");
LogPrint(BCLog::LogFlags::WALLETDB, "Flushing wallet.dat");
nLastFlushed = nWalletDBUpdated;
int64_t nStart = GetTimeMillis();

Expand All @@ -666,7 +666,7 @@ void ThreadFlushWalletDB(void* parg)
// This is handled in CDB::Flush, which has a while loop that also does in the right place what
// the intention of the below line was.
// bitdb.mapFileUseCount.erase(mi++);
if (fDebug10) LogPrintf("Flushed wallet.dat %" PRId64 "ms", GetTimeMillis() - nStart);
LogPrint(BCLog::LogFlags::WALLETDB, "Flushed wallet.dat %" PRId64 "ms", GetTimeMillis() - nStart);
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/walletdb.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class CKeyMetadata
class CWalletDB : public CDB
{
public:
CWalletDB(std::string strFilename, const char* pszMode="r+") : CDB(strFilename.c_str(), pszMode)
CWalletDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnClose = true) : CDB(strFilename, pszMode, fFlushOnClose)
{
}
private:
Expand Down

0 comments on commit 9a56c82

Please sign in to comment.