diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index b30b7cf7a2e2..8fb8ce14b261 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -37,8 +37,7 @@ def setup_chain(self): initialize_chain(self.options.tmpdir) def setup_network(self, split=False): - self.nodes = start_nodes(2, self.options.tmpdir) - connect_nodes_bi(self.nodes, 0, 1) + self.nodes = start_nodes(1, self.options.tmpdir) self.is_network_split = False self.sync_all() @@ -56,9 +55,33 @@ def _test_gettxoutsetinfo(self): assert_equal(res[u'height'], 200) assert_equal(res[u'txouts'], 200) assert_equal(res[u'bytes_serialized'], 14273), + assert_equal(res['bestblock'], node.getblockhash(200)) assert_equal(len(res[u'bestblock']), 64) assert_equal(len(res[u'hash_serialized']), 64) + print("Test that gettxoutsetinfo() works for blockchain with just the genesis block") + b1hash = node.getblockhash(1) + node.invalidateblock(b1hash) + + res2 = node.gettxoutsetinfo() + assert_equal(res2['transactions'], 0) + assert_equal(res2['total_amount'], Decimal('0')) + assert_equal(res2['height'], 0) + assert_equal(res2['txouts'], 0) + assert_equal(res2['bestblock'], node.getblockhash(0)) + assert_equal(len(res2['hash_serialized']), 64) + + print("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") + node.reconsiderblock(b1hash) + + res3 = node.gettxoutsetinfo() + assert_equal(res['total_amount'], res3['total_amount']) + assert_equal(res['transactions'], res3['transactions']) + assert_equal(res['height'], res3['height']) + assert_equal(res['txouts'], res3['txouts']) + assert_equal(res['bestblock'], res3['bestblock']) + assert_equal(res['hash_serialized'], res3['hash_serialized']) + def _test_getblockheader(self): node = self.nodes[0] diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 09c68fbe550c..4a4bd41b3950 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -14,6 +14,64 @@ #include #include #include +#include + +class CBitcoinLevelDBLogger : public leveldb::Logger { +public: + // This code is adapted from posix_logger.h, which is why it is using vsprintf. + // Please do not do this in normal code + virtual void Logv(const char * format, va_list ap) override { + if (!LogAcceptCategory("leveldb")) + return; + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } + else { + bufsize = 30000; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + // Do not use vsnprintf elsewhere in bitcoin source code, see above. + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } + else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + base[std::min(bufsize - 1, (int)(p - base))] = '\0'; + LogPrintStr(base); + if (base != buffer) { + delete[] base; + } + break; + } + } +}; static leveldb::Options GetOptions(size_t nCacheSize) { @@ -23,6 +81,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) options.filter_policy = leveldb::NewBloomFilterPolicy(10); options.compression = leveldb::kNoCompression; options.max_open_files = 64; + options.info_log = new CBitcoinLevelDBLogger(); if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) { // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error // on corruption in later versions. @@ -82,6 +141,8 @@ CDBWrapper::~CDBWrapper() pdb = NULL; delete options.filter_policy; options.filter_policy = NULL; + delete options.info_log; + options.info_log = NULL; delete options.block_cache; options.block_cache = NULL; delete penv; diff --git a/src/init.cpp b/src/init.cpp index 4806dddd2efa..f49ccc9bf4aa 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -544,7 +544,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitdescendantcount=", strprintf("Do not accept transactions if any ancestor would have or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantsize=", strprintf("Do not accept transactions if any ancestor would have more than kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); } - string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq, " + string debugCategories = "addrman, alert, bench, coindb, db, http, leveldb, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq, " "dash (or specifically: gobject, instantsend, keepass, masternode, mnpayments, mnsync, privatesend, spork)"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; diff --git a/src/serialize.h b/src/serialize.h index cf4267f6b47c..eecb86652e9d 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -386,6 +386,7 @@ I ReadVarInt(Stream& is) #define FLATDATA(obj) REF(CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj))) #define VARINT(obj) REF(WrapVarInt(REF(obj))) +#define COMPACTSIZE(obj) REF(CCompactSize(REF(obj))) #define LIMITED_STRING(obj,n) REF(LimitedString< n >(REF(obj))) /** @@ -456,6 +457,28 @@ class CVarInt } }; +class CCompactSize +{ +protected: + uint64_t &n; +public: + CCompactSize(uint64_t& nIn) : n(nIn) { } + + unsigned int GetSerializeSize(int, int) const { + return GetSizeOfCompactSize(n); + } + + template + void Serialize(Stream &s, int, int) const { + WriteCompactSize(s, n); + } + + template + void Unserialize(Stream& s, int, int) { + n = ReadCompactSize(s); + } +}; + template class LimitedString { diff --git a/src/txdb.cpp b/src/txdb.cpp index ad6e374f6fe1..cbfcc01b789e 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -104,7 +104,11 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const that restriction. */ i->pcursor->Seek(DB_COINS); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (i->pcursor->Valid()) { + i->pcursor->GetKey(i->keyTmp); + } else { + i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false + } return i; } diff --git a/src/txdb.h b/src/txdb.h index cbfd336a0a0c..27d21b6778e7 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -22,8 +22,14 @@ class CBlockIndex; class CCoinsViewDBCursor; class uint256; +//! Compensate for extra memory peak (x1.5-x1.9) at flush time. +static constexpr int DB_PEAK_USAGE_FACTOR = 2; +//! No need to periodic flush if at least this much space still available. +static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; +//! Always periodic flush if less than this much space still available. +static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) -static const int64_t nDefaultDbCache = 300; +static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache (MiB) diff --git a/src/validation.cpp b/src/validation.cpp index f83c325e4718..a93d84feb678 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1685,24 +1685,36 @@ bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint return fClean; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +enum DisconnectResult { - assert(pindex->GetBlockHash() == view.GetBestBlock()); + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; - if (pfClean) - *pfClean = false; +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */ +static DisconnectResult DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view) +{ + assert(pindex->GetBlockHash() == view.GetBestBlock()); bool fClean = true; CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) - return error("DisconnectBlock(): no undo data available"); - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock(): failure reading undo data"); + if (pos.IsNull()) { + error("DisconnectBlock(): no undo data available"); + return DISCONNECT_FAILED; + } + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } - if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock(): block and undo data inconsistent"); + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } std::vector > addressIndex; std::vector > addressUnspentIndex; @@ -1766,8 +1778,10 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // restore inputs if (i > 0) { // not coinbases const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; const CTxInUndo &undo = txundo.vprevout[j]; @@ -1815,21 +1829,18 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); - if (pfClean) { - *pfClean = fClean; - return true; - } - if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { - return AbortNode(state, "Failed to delete address index"); + AbortNode(state, "Failed to delete address index"); + return DISCONNECT_FAILED; } if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { - return AbortNode(state, "Failed to write address unspent index"); + AbortNode(state, "Failed to write address unspent index"); + return DISCONNECT_FAILED; } } - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } void static FlushBlockFile(bool fFinalize = false) @@ -1945,7 +1956,10 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck = false) { const CChainParams& chainparams = Params(); AssertLockHeld(cs_main); @@ -2333,10 +2347,11 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { nLastSetChain = nNow; } int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 100 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - 100 * 1024 * 1024); + // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), + std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -2481,7 +2496,7 @@ bool static DisconnectTip(CValidationState& state, const Consensus::Params& cons int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (DisconnectBlock(block, state, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); assert(view.Flush()); } @@ -3875,15 +3890,17 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { - bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + DisconnectResult res = DisconnectBlock(block, state, pindex, coins); + if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } pindexState = pindex->pprev; - if (!fClean) { + if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; - } else + } else { nGoodTransactions += block.vtx.size(); + } } if (ShutdownRequested()) return true; diff --git a/src/validation.h b/src/validation.h index 80c8f284b7c7..c372c7a6ab25 100644 --- a/src/validation.h +++ b/src/validation.h @@ -429,19 +429,10 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus /** Functions for validating blocks and updating the block tree */ -/** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& coins, bool* pfClean = NULL); - /** Reprocess a number of blocks to try and get on the correct chain again **/ bool DisconnectBlocks(int blocks); void ReprocessBlocks(int nBlocks); -/** Apply the effects of this block (with given index) on the UTXO set represented by coins */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, bool fJustCheck = false); - /** Context-independent validity checks */ bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true);