diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 000000000000..9529641f63a8 --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh ' + echo + echo " if is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/dash-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/dash-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/src/Makefile.am b/src/Makefile.am index 3818fda8753f..4ce5d65836a0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -238,6 +238,7 @@ BITCOIN_CORE_H = \ node/coinstats.h \ node/context.h \ node/transaction.h \ + node/utxo_snapshot.h \ noui.h \ optional.h \ policy/feerate.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 8b12e3048423..0b5b258c0dff 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -229,6 +229,7 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_chainstate_tests.cpp \ test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ test/versionbits_tests.cpp diff --git a/src/coins.cpp b/src/coins.cpp index 12a390db6984..189369715513 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -257,6 +257,14 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } +void CCoinsViewCache::ReallocateCache() +{ + // Cache should be empty when we're calling this. + assert(cacheCoins.size() == 0); + cacheCoins.~CCoinsMap(); + ::new (&cacheCoins) CCoinsMap(); +} + static const size_t MAX_OUTPUTS_PER_BLOCK = MaxBlockSize() / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) diff --git a/src/coins.h b/src/coins.h index ba20acd9d330..14d6e9177b3f 100644 --- a/src/coins.h +++ b/src/coins.h @@ -331,6 +331,13 @@ class CCoinsViewCache : public CCoinsViewBacked //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; + //! Force a reallocation of the cache map. This is required when downsizing + //! the cache because the map's allocator may be hanging onto a lot of + //! memory despite having called .clear(). + //! + //! See: https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory + void ReallocateCache(); + private: /** * @note this is marked const, but may actually append to `cacheCoins`, increasing diff --git a/src/governance/object.cpp b/src/governance/object.cpp index 704af231acec..84291a308a0b 100644 --- a/src/governance/object.cpp +++ b/src/governance/object.cpp @@ -533,12 +533,10 @@ bool CGovernanceObject::IsCollateralValid(std::string& strError, bool& fMissingC fMissingConfirmations = false; uint256 nExpectedHash = GetHash(); - CTransactionRef txCollateral; - uint256 nBlockHash; - // RETRIEVE TRANSACTION IN QUESTION - - if (!GetTransaction(nCollateralHash, txCollateral, Params().GetConsensus(), nBlockHash)) { + uint256 nBlockHash; + CTransactionRef txCollateral = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, nCollateralHash, Params().GetConsensus(), nBlockHash); + if (!txCollateral) { strError = strprintf("Can't find collateral tx %s", nCollateralHash.ToString()); LogPrintf("CGovernanceObject::IsCollateralValid -- %s\n", strError); return false; diff --git a/src/init.cpp b/src/init.cpp index 92080701ee87..b929d965d614 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1056,13 +1056,14 @@ static bool InitSanityCheck() return true; } -static bool AppInitServers(const util::Ref& context) +static bool AppInitServers(const util::Ref& context, NodeContext& node) { RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) return false; StartRPC(); + node.rpc_interruption_point = RpcInterruptionPoint; if (!StartHTTPRPC(context)) return false; if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); @@ -1748,7 +1749,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA if (gArgs.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); - if (!AppInitServers(context)) + if (!AppInitServers(context, node)) return InitError(_("Unable to start HTTP server. See debug log for details.")); } @@ -1949,7 +1950,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; - nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache + int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nEvoDbCache = 1024 * 1024 * 16; // TODO LogPrintf("Cache configuration:\n"); @@ -1982,7 +1983,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA try { LOCK(cs_main); chainman.InitializeChainstate(); - UnloadBlockIndex(); + chainman.m_total_coinstip_cache = nCoinCacheUsage; + chainman.m_total_coinsdb_cache = nCoinDBCache; + + UnloadBlockIndex(node.mempool); // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -1997,7 +2001,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA llmq::quorumSnapshotManager.reset(); llmq::quorumSnapshotManager.reset(new llmq::CQuorumSnapshotManager(*evoDb)); - llmq::InitLLMQSystem(*evoDb, *node.connman, false, fReset || fReindexChainState); + llmq::InitLLMQSystem(*evoDb, *node.mempool, *node.connman, false, fReset || fReindexChainState); if (fReset) { pblocktree->WriteReindexing(true); @@ -2100,7 +2104,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(); + chainstate->InitCoinsCache(nCoinCacheUsage); assert(chainstate->CanFlushToDisk()); // flush evodb diff --git a/src/llmq/chainlocks.cpp b/src/llmq/chainlocks.cpp index 96d3435bc03b..697c2d7b461b 100644 --- a/src/llmq/chainlocks.cpp +++ b/src/llmq/chainlocks.cpp @@ -23,9 +23,9 @@ namespace llmq { CChainLocksHandler* chainLocksHandler; -CChainLocksHandler::CChainLocksHandler(CConnman& _connman) : +CChainLocksHandler::CChainLocksHandler(CTxMemPool& _mempool, CConnman& _connman) : scheduler(std::make_unique()), - connman(_connman) + mempool(_mempool), connman(_connman) { CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, scheduler.get()); scheduler_thread = std::make_unique(std::bind(&TraceThread, "cl-schdlr", serviceLoop)); @@ -647,9 +647,9 @@ void CChainLocksHandler::Cleanup() } } for (auto it = txFirstSeenTime.begin(); it != txFirstSeenTime.end(); ) { - CTransactionRef tx; uint256 hashBlock; - if (!GetTransaction(it->first, tx, Params().GetConsensus(), hashBlock)) { + CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, it->first, Params().GetConsensus(), hashBlock); + if (!tx) { // tx has vanished, probably due to conflicts it = txFirstSeenTime.erase(it); } else if (!hashBlock.IsNull()) { diff --git a/src/llmq/chainlocks.h b/src/llmq/chainlocks.h index b472342626fb..410e24930069 100644 --- a/src/llmq/chainlocks.h +++ b/src/llmq/chainlocks.h @@ -22,6 +22,7 @@ class CConnman; class CBlockIndex; class CScheduler; +class CTxMemPool; namespace llmq { @@ -36,6 +37,7 @@ class CChainLocksHandler : public CRecoveredSigsListener private: CConnman& connman; + CTxMemPool& mempool; std::unique_ptr scheduler; std::unique_ptr scheduler_thread; mutable CCriticalSection cs; @@ -68,7 +70,7 @@ class CChainLocksHandler : public CRecoveredSigsListener int64_t lastCleanupTime GUARDED_BY(cs) {0}; public: - explicit CChainLocksHandler(CConnman& _connman); + explicit CChainLocksHandler(CTxMemPool& _mempool, CConnman& _connman); ~CChainLocksHandler(); void Start(); diff --git a/src/llmq/init.cpp b/src/llmq/init.cpp index 3f96a8d7abfc..edccb638ec7f 100644 --- a/src/llmq/init.cpp +++ b/src/llmq/init.cpp @@ -23,7 +23,7 @@ namespace llmq CBLSWorker* blsWorker; -void InitLLMQSystem(CEvoDB& evoDb, CConnman& connman, bool unitTests, bool fWipe) +void InitLLMQSystem(CEvoDB& evoDb, CTxMemPool& mempool, CConnman& connman, bool unitTests, bool fWipe) { blsWorker = new CBLSWorker(); @@ -33,8 +33,8 @@ void InitLLMQSystem(CEvoDB& evoDb, CConnman& connman, bool unitTests, bool fWipe quorumManager = new CQuorumManager(evoDb, connman, *blsWorker, *quorumDKGSessionManager); quorumSigSharesManager = new CSigSharesManager(connman); quorumSigningManager = new CSigningManager(connman, unitTests, fWipe); - chainLocksHandler = new CChainLocksHandler(connman); - quorumInstantSendManager = new CInstantSendManager(connman, unitTests, fWipe); + chainLocksHandler = new CChainLocksHandler(mempool, connman); + quorumInstantSendManager = new CInstantSendManager(mempool, connman, unitTests, fWipe); // NOTE: we use this only to wipe the old db, do NOT use it for anything else // TODO: remove it in some future version diff --git a/src/llmq/init.h b/src/llmq/init.h index c56c5878e423..1f42d3fb2e22 100644 --- a/src/llmq/init.h +++ b/src/llmq/init.h @@ -8,12 +8,13 @@ class CConnman; class CDBWrapper; class CEvoDB; +class CTxMemPool; namespace llmq { // Init/destroy LLMQ globals -void InitLLMQSystem(CEvoDB& evoDb, CConnman& connman, bool unitTests, bool fWipe = false); +void InitLLMQSystem(CEvoDB& evoDb, CTxMemPool& mempool, CConnman& connman, bool unitTests, bool fWipe = false); void DestroyLLMQSystem(); // Manage scheduled tasks, threads, listeners etc. diff --git a/src/llmq/instantsend.cpp b/src/llmq/instantsend.cpp index 8bf9d0603dfa..4569e922452f 100644 --- a/src/llmq/instantsend.cpp +++ b/src/llmq/instantsend.cpp @@ -64,8 +64,6 @@ void CInstantSendDb::Upgrade() if (!db->Read(DB_VERSION, v) || v < CInstantSendDb::CURRENT_VERSION) { CDBBatch batch(*db); CInstantSendLock islock; - CTransactionRef tx; - uint256 hashBlock; auto it = std::unique_ptr(db->NewIterator()); auto firstKey = std::make_tuple(DB_ISLOCK_BY_HASH, uint256()); @@ -76,7 +74,9 @@ void CInstantSendDb::Upgrade() if (!it->GetKey(curKey) || std::get<0>(curKey) != DB_ISLOCK_BY_HASH) { break; } - if (it->GetValue(islock) && !GetTransaction(islock.txid, tx, Params().GetConsensus(), hashBlock)) { + uint256 hashBlock; + CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, islock.txid, Params().GetConsensus(), hashBlock); + if (it->GetValue(islock) && !tx) { // Drop locks for unknown txes batch.Erase(std::make_tuple(DB_HASH_BY_TXID, islock.txid)); for (auto& in : islock.inputs) { @@ -603,10 +603,10 @@ bool CInstantSendManager::CheckCanLock(const COutPoint& outpoint, bool printDebu return false; } - CTransactionRef tx; uint256 hashBlock; + CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, outpoint.hash, params, hashBlock); // this relies on enabled txindex and won't work if we ever try to remove the requirement for txindex for masternodes - if (!GetTransaction(outpoint.hash, tx, params, hashBlock)) { + if (!tx) { if (printDebug) { LogPrint(BCLog::INSTANTSEND, "CInstantSendManager::%s -- txid=%s: failed to find parent TX %s\n", __func__, txHash.ToString(), outpoint.hash.ToString()); @@ -661,9 +661,9 @@ void CInstantSendManager::HandleNewInputLockRecoveredSig(const CRecoveredSig& re g_txindex->BlockUntilSyncedToCurrentChain(); } - CTransactionRef tx; uint256 hashBlock; - if (!GetTransaction(txid, tx, Params().GetConsensus(), hashBlock)) { + CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, txid, Params().GetConsensus(), hashBlock); + if (!tx) { return; } @@ -1048,11 +1048,11 @@ void CInstantSendManager::ProcessInstantSendLock(NodeId from, const uint256& has return; } - CTransactionRef tx; uint256 hashBlock; + CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, islock->txid, Params().GetConsensus(), hashBlock); const CBlockIndex* pindexMined{nullptr}; // we ignore failure here as we must be able to propagate the lock even if we don't have the TX locally - if (GetTransaction(islock->txid, tx, Params().GetConsensus(), hashBlock) && !hashBlock.IsNull()) { + if (tx && !hashBlock.IsNull()) { pindexMined = WITH_LOCK(cs_main, return LookupBlockIndex(hashBlock)); // Let's see if the TX that was locked by this islock is already mined in a ChainLocked block. If yes, diff --git a/src/llmq/instantsend.h b/src/llmq/instantsend.h index ef09e721d535..c3d97d269def 100644 --- a/src/llmq/instantsend.h +++ b/src/llmq/instantsend.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -194,6 +195,7 @@ class CInstantSendManager : public CRecoveredSigsListener private: CInstantSendDb db; CConnman& connman; + CTxMemPool& mempool; std::atomic fUpgradedDB{false}; @@ -241,7 +243,7 @@ class CInstantSendManager : public CRecoveredSigsListener std::unordered_set pendingRetryTxs GUARDED_BY(cs_pendingRetry); public: - explicit CInstantSendManager(CConnman& _connman, bool unitTests, bool fWipe) : db(unitTests, fWipe), connman(_connman) { workInterrupt.reset(); } + explicit CInstantSendManager(CTxMemPool& _mempool, CConnman& _connman, bool unitTests, bool fWipe) : db(unitTests, fWipe), mempool(_mempool), connman(_connman) { workInterrupt.reset(); } ~CInstantSendManager() = default; void Start(); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 38a331bf0852..3d6aa9fe81d1 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2950,7 +2950,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec pfrom->GetId()); } else if (!fAlreadyHave) { bool allowWhileInIBD = allowWhileInIBDObjs.count(inv.type); - if (allowWhileInIBD || (!fImporting && !fReindex && !::ChainstateActive().IsInitialBlockDownload())) { + if (allowWhileInIBD || !chainman.ActiveChainstate().IsInitialBlockDownload()) { RequestObject(State(pfrom->GetId()), inv, current_time); } } diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 33149e3f4df2..d7e0a9ca1269 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -85,8 +85,9 @@ static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash, //! Calculate statistics about the unspent transaction output set template -static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj) +static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const std::function& interruption_point) { + stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); assert(pcursor); @@ -101,7 +102,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj) uint256 prevkey; std::map outputs; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); + interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { @@ -111,6 +112,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } @@ -126,19 +128,19 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj) return true; } -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type) +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function& interruption_point) { switch (hash_type) { case(CoinStatsHashType::HASH_SERIALIZED): { CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - return GetUTXOStats(view, stats, ss); + return GetUTXOStats(view, stats, ss, interruption_point); } case(CoinStatsHashType::MUHASH): { MuHash3072 muhash; - return GetUTXOStats(view, stats, muhash); + return GetUTXOStats(view, stats, muhash, interruption_point); } case(CoinStatsHashType::NONE): { - return GetUTXOStats(view, stats, nullptr); + return GetUTXOStats(view, stats, nullptr, interruption_point); } } // no default case, so the compiler can warn about missing cases assert(false); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7e24193ccac4..997bab87f137 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -10,6 +10,7 @@ #include #include +#include class CCoinsView; @@ -21,19 +22,20 @@ enum class CoinStatsHashType { struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + CAmount nTotalAmount{0}; + + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const CoinStatsHashType hash_type); +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index 31a59f2403a4..7cbd4ab24ff0 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -6,6 +6,7 @@ #define BITCOIN_NODE_CONTEXT_H #include +#include #include #include @@ -39,6 +40,7 @@ struct NodeContext { std::unique_ptr chain; std::vector> chain_clients; std::unique_ptr scheduler; + std::function rpc_interruption_point = [] {}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 000000000000..fe78cb46bde0 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,41 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include +#include + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count, obj.m_nchaintx); } +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 08c592d74ea7..d336cc318ad6 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -87,7 +87,7 @@ void AppTests::appTests() // Reset global state to avoid interfering with later tests. LogInstance().DisconnectTestLogger(); AbortShutdown(); - UnloadBlockIndex(); + UnloadBlockIndex(/* mempool */ nullptr); WITH_LOCK(::cs_main, g_chainman.Reset()); } diff --git a/src/rest.cpp b/src/rest.cpp index 4a22eef48fa2..ed5585318417 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -68,13 +68,32 @@ static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string me } /** - * Get the node context mempool. + * Get the node context. * - * Set the HTTP error and return nullptr if node context - * mempool is not found. + * @param[in] req The HTTP request, whose status code will be set if node + * context is not found. + * @returns Pointer to the node context or nullptr if not found. + */ +static NodeContext* GetNodeContext(const util::Ref& context, HTTPRequest* req) +{ + NodeContext* node = context.Has() ? &context.Get() : nullptr; + if (!node) { + RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, + strprintf("%s:%d (%s)\n" + "Internal bug detected: Node context not found!\n" + "You may report this issue here: %s\n", + __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); + return nullptr; + } + return node; +} + +/** + * Get the node context mempool. * - * @param[in] req the HTTP request - * return pointer to the mempool or nullptr if no mempool found + * @param[in] req The HTTP request, whose status code will be set if node + * context mempool is not found. + * @returns Pointer to the mempool or nullptr if no mempool found. */ static CTxMemPool* GetMemPool(const util::Ref& context, HTTPRequest* req) { @@ -371,10 +390,13 @@ static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::strin g_txindex->BlockUntilSyncedToCurrentChain(); } - CTransactionRef tx; + const NodeContext* const node = GetNodeContext(context, req); + if (!node) return false; uint256 hashBlock = uint256(); - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock)) + const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, node->mempool, hash, Params().GetConsensus(), hashBlock); + if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); + } switch (rf) { case RetFormat::BINARY: { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index afeebbe74583..3eceeb919ef8 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -1253,7 +1254,8 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); - if (GetUTXOStats(coins_view, stats, hash_type)) { + NodeContext& node = EnsureNodeContext(request.context); + if (GetUTXOStats(coins_view, stats, hash_type, node.rpc_interruption_point)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); @@ -2357,8 +2359,10 @@ static UniValue savemempool(const JSONRPCRequest& request) return NullUniValue; } +namespace { //! Search for a given set of pubkey scripts -bool FindScriptPubKey(std::atomic& scan_progress, const std::atomic& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set& needles, std::map& out_results) { +bool FindScriptPubKey(std::atomic& scan_progress, const std::atomic& should_abort, int64_t& count, CCoinsViewCursor* cursor, const std::set& needles, std::map& out_results, std::function& interruption_point) +{ scan_progress = 0; count = 0; while (cursor->Valid()) { @@ -2366,7 +2370,7 @@ bool FindScriptPubKey(std::atomic& scan_progress, const std::atomic& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - boost::this_thread::interruption_point(); + interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -2385,6 +2389,7 @@ bool FindScriptPubKey(std::atomic& scan_progress, const std::atomic& scan_progress = 100; return true; } +} // namespace /** RAII object to prevent concurrency issue when scanning the txout set */ static std::mutex g_utxosetscan; @@ -2567,7 +2572,8 @@ UniValue scantxoutset(const JSONRPCRequest& request) tip = ::ChainActive().Tip(); CHECK_NONFATAL(tip); } - bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); + NodeContext& node = EnsureNodeContext(request.context); + bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins, node.rpc_interruption_point); result.pushKV("success", res); result.pushKV("txouts", count); result.pushKV("height", tip->nHeight); @@ -2673,6 +2679,112 @@ static UniValue getblockfilter(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "coins_written", "the number of coins written in the snapshot"}, + {RPCResult::Type::STR_HEX, "base_hash", "the hash of the base of the snapshot"}, + {RPCResult::Type::NUM, "base_height", "the height of the base of the snapshot"}, + {RPCResult::Type::STR, "path", "the absolute path that the snapshot was written to"}, + } + }, + RPCExamples{ + HelpExampleCli("dumptxoutset", "utxo.dat") + } + }.Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE* file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr pcursor; + CCoinsStats stats; + CBlockIndex* tip; + NodeContext& node = EnsureNodeContext(request.context); + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats + // based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr(::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0) node.rpc_interruption_point(); + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2714,6 +2826,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp index a3f40b1df46d..fe9b33baf30d 100644 --- a/src/rpc/masternode.cpp +++ b/src/rpc/masternode.cpp @@ -416,15 +416,15 @@ static UniValue masternode_payments(const JSONRPCRequest& request) // Note: we have to actually calculate block reward from scratch instead of simply querying coinbase vout // because miners might collect less coins than they potentially could and this would break our calculations. CAmount nBlockFees{0}; + NodeContext& node = EnsureNodeContext(request.context); for (const auto& tx : block.vtx) { if (tx->IsCoinBase()) { continue; } CAmount nValueIn{0}; for (const auto& txin : tx->vin) { - CTransactionRef txPrev; uint256 blockHashTmp; - GetTransaction(txin.prevout.hash, txPrev, Params().GetConsensus(), blockHashTmp); + CTransactionRef txPrev = GetTransaction(/* block_index */ nullptr, node.mempool, txin.prevout.hash, Params().GetConsensus(), blockHashTmp); nValueIn += txPrev->vout[txin.prevout.n].nValue; } nBlockFees += nValueIn - tx->GetValueOut(); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e84e3088574b..5b35a44d2e0b 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -201,6 +201,8 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) }, }.Check(request); + const NodeContext& node = EnsureNodeContext(request.context); + bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); CBlockIndex* blockindex = nullptr; @@ -232,9 +234,9 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain(); } - CTransactionRef tx; uint256 hash_block; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, blockindex)) { + const CTransactionRef tx = GetTransaction(blockindex, node.mempool, hash, Params().GetConsensus(), hash_block); + if (!tx) { std::string errmsg; if (blockindex) { if (!(blockindex->nStatus & BLOCK_HAVE_DATA)) { @@ -292,11 +294,13 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) UniValue txids = request.params[0].get_array(); for (unsigned int idx = 0; idx < txids.size(); idx++) { const UniValue& txid = txids[idx]; - if (txid.get_str().length() != 64 || !IsHex(txid.get_str())) + if (txid.get_str().length() != 64 || !IsHex(txid.get_str())) { throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid txid ")+txid.get_str()); + } uint256 hash(uint256S(txid.get_str())); - if (setTxids.count(hash)) + if (setTxids.count(hash)) { throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ")+txid.get_str()); + } setTxids.insert(hash); oneTxid = hash; } @@ -331,11 +335,11 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) LOCK(cs_main); - if (pblockindex == nullptr) - { - CTransactionRef tx; - if (!GetTransaction(oneTxid, tx, Params().GetConsensus(), hashBlock) || hashBlock.IsNull()) + if (pblockindex == nullptr) { + const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, oneTxid, Params().GetConsensus(), hashBlock); + if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); + } pblockindex = LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); @@ -343,15 +347,19 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) } CBlock block; - if(!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); + } unsigned int ntxFound = 0; - for (const auto& tx : block.vtx) - if (setTxids.count(tx->GetHash())) + for (const auto& tx : block.vtx) { + if (setTxids.count(tx->GetHash())) { ntxFound++; - if (ntxFound != setTxids.size()) + } + } + if (ntxFound != setTxids.size()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Not all transactions found in specified or retrieved block"); + } CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION); CMerkleBlock mb(block, setTxids); diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index 5cf93616e86e..681d5f0710a3 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -955,9 +955,9 @@ static UniValue BuildDMNListEntry(CWallet* pwallet, const CDeterministicMN& dmn, bool hasVotingKey = CheckWalletOwnsKey(pwallet, dmn.pdmnState->keyIDVoting); bool ownsCollateral = false; - CTransactionRef collateralTx; uint256 tmpHashBlock; - if (GetTransaction(dmn.collateralOutpoint.hash, collateralTx, Params().GetConsensus(), tmpHashBlock)) { + CTransactionRef collateralTx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, dmn.collateralOutpoint.hash, Params().GetConsensus(), tmpHashBlock); + if (collateralTx) { ownsCollateral = CheckWalletOwnsScript(pwallet, collateralTx->vout[dmn.collateralOutpoint.n].scriptPubKey); } diff --git a/src/rpc/rpcquorums.cpp b/src/rpc/rpcquorums.cpp index 2dc5c3469503..4f3ad4fe0ab1 100644 --- a/src/rpc/rpcquorums.cpp +++ b/src/rpc/rpcquorums.cpp @@ -809,9 +809,9 @@ static UniValue verifyislock(const JSONRPCRequest& request) CBlockIndex* pindexMined{nullptr}; { LOCK(cs_main); - CTransactionRef tx; uint256 hash_block; - if (GetTransaction(txid, tx, Params().GetConsensus(), hash_block) && !hash_block.IsNull()) { + CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, txid, Params().GetConsensus(), hash_block); + if (tx && !hash_block.IsNull()) { pindexMined = LookupBlockIndex(hash_block); } } diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 96003b735ded..c859fa7c763a 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -13,9 +13,9 @@ #include #include -#include #include #include +#include #include #include // for unique_ptr @@ -340,6 +340,11 @@ bool IsRPCRunning() return fRPCRunning; } +void RpcInterruptionPoint() +{ + if (!IsRPCRunning()) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); +} + void SetRPCWarmupStatus(const std::string& newStatus) { LOCK(cs_rpcWarmup); diff --git a/src/rpc/server.h b/src/rpc/server.h index 4de2724f19a7..023bbfbe0034 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -10,10 +10,10 @@ #include #include +#include #include #include #include -#include #include @@ -28,6 +28,9 @@ namespace RPCServer /** Query whether RPC is running */ bool IsRPCRunning(); +/** Throw JSONRPCError if RPC is not running */ +void RpcInterruptionPoint(); + /** * Set the RPC warmup status. When this is done, all RPC calls will error out * immediately with RPC_IN_WARMUP. diff --git a/src/test/block_reward_reallocation_tests.cpp b/src/test/block_reward_reallocation_tests.cpp index f360f214071e..804eb95e8aac 100644 --- a/src/test/block_reward_reallocation_tests.cpp +++ b/src/test/block_reward_reallocation_tests.cpp @@ -94,20 +94,20 @@ static void FundTransaction(CMutableTransaction& tx, SimpleUTXOMap& utoxs, const } } -static void SignTransaction(CMutableTransaction& tx, const CKey& coinbaseKey) +static void SignTransaction(const CTxMemPool& mempool, CMutableTransaction& tx, const CKey& coinbaseKey) { CBasicKeyStore tempKeystore; tempKeystore.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); for (size_t i = 0; i < tx.vin.size(); i++) { - CTransactionRef txFrom; uint256 hashBlock; - BOOST_ASSERT(GetTransaction(tx.vin[i].prevout.hash, txFrom, Params().GetConsensus(), hashBlock)); + CTransactionRef txFrom = GetTransaction(/* block_index */ nullptr, &mempool, tx.vin[i].prevout.hash, Params().GetConsensus(), hashBlock); + BOOST_ASSERT(txFrom); BOOST_ASSERT(SignSignature(tempKeystore, *txFrom, tx, i, SIGHASH_ALL)); } } -static CMutableTransaction CreateProRegTx(SimpleUTXOMap& utxos, int port, const CScript& scriptPayout, const CKey& coinbaseKey, CKey& ownerKeyRet, CBLSSecretKey& operatorKeyRet) +static CMutableTransaction CreateProRegTx(const CTxMemPool& mempool, SimpleUTXOMap& utxos, int port, const CScript& scriptPayout, const CKey& coinbaseKey, CKey& ownerKeyRet, CBLSSecretKey& operatorKeyRet) { ownerKeyRet.MakeNewKey(true); operatorKeyRet.MakeNewKey(); @@ -126,7 +126,7 @@ static CMutableTransaction CreateProRegTx(SimpleUTXOMap& utxos, int port, const FundTransaction(tx, utxos, scriptPayout, 1000 * COIN); proTx.inputsHash = CalcTxInputsHash(CTransaction(tx)); SetTxPayload(tx, proTx); - SignTransaction(tx, coinbaseKey); + SignTransaction(mempool, tx, coinbaseKey); return tx; } @@ -187,7 +187,7 @@ BOOST_FIXTURE_TEST_CASE(block_reward_reallocation, TestChainBRRBeforeActivationS CKey ownerKey; CBLSSecretKey operatorKey; auto utxos = BuildSimpleUtxoMap(m_coinbase_txns); - auto tx = CreateProRegTx(utxos, 1, GenerateRandomAddress(), coinbaseKey, ownerKey, operatorKey); + auto tx = CreateProRegTx(*m_node.mempool, utxos, 1, GenerateRandomAddress(), coinbaseKey, ownerKey, operatorKey); CreateAndProcessBlock({tx}, coinbaseKey); diff --git a/src/test/evo_deterministicmns_tests.cpp b/src/test/evo_deterministicmns_tests.cpp index c0406dacdd05..a7372a8ae41e 100644 --- a/src/test/evo_deterministicmns_tests.cpp +++ b/src/test/evo_deterministicmns_tests.cpp @@ -80,20 +80,20 @@ static void FundTransaction(CMutableTransaction& tx, SimpleUTXOMap& utoxs, const } } -static void SignTransaction(CMutableTransaction& tx, const CKey& coinbaseKey) +static void SignTransaction(const CTxMemPool& mempool, CMutableTransaction& tx, const CKey& coinbaseKey) { CBasicKeyStore tempKeystore; tempKeystore.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); for (size_t i = 0; i < tx.vin.size(); i++) { - CTransactionRef txFrom; uint256 hashBlock; - BOOST_ASSERT(GetTransaction(tx.vin[i].prevout.hash, txFrom, Params().GetConsensus(), hashBlock)); + CTransactionRef txFrom = GetTransaction(/* block_index */ nullptr, &mempool, tx.vin[i].prevout.hash, Params().GetConsensus(), hashBlock); + BOOST_ASSERT(txFrom); BOOST_ASSERT(SignSignature(tempKeystore, *txFrom, tx, i, SIGHASH_ALL)); } } -static CMutableTransaction CreateProRegTx(SimpleUTXOMap& utxos, int port, const CScript& scriptPayout, const CKey& coinbaseKey, CKey& ownerKeyRet, CBLSSecretKey& operatorKeyRet) +static CMutableTransaction CreateProRegTx(const CTxMemPool& mempool, SimpleUTXOMap& utxos, int port, const CScript& scriptPayout, const CKey& coinbaseKey, CKey& ownerKeyRet, CBLSSecretKey& operatorKeyRet) { ownerKeyRet.MakeNewKey(true); operatorKeyRet.MakeNewKey(); @@ -112,12 +112,12 @@ static CMutableTransaction CreateProRegTx(SimpleUTXOMap& utxos, int port, const FundTransaction(tx, utxos, scriptPayout, 1000 * COIN, coinbaseKey); proTx.inputsHash = CalcTxInputsHash(CTransaction(tx)); SetTxPayload(tx, proTx); - SignTransaction(tx, coinbaseKey); + SignTransaction(mempool, tx, coinbaseKey); return tx; } -static CMutableTransaction CreateProUpServTx(SimpleUTXOMap& utxos, const uint256& proTxHash, const CBLSSecretKey& operatorKey, int port, const CScript& scriptOperatorPayout, const CKey& coinbaseKey) +static CMutableTransaction CreateProUpServTx(const CTxMemPool& mempool, SimpleUTXOMap& utxos, const uint256& proTxHash, const CBLSSecretKey& operatorKey, int port, const CScript& scriptOperatorPayout, const CKey& coinbaseKey) { CProUpServTx proTx; proTx.proTxHash = proTxHash; @@ -131,12 +131,12 @@ static CMutableTransaction CreateProUpServTx(SimpleUTXOMap& utxos, const uint256 proTx.inputsHash = CalcTxInputsHash(CTransaction(tx)); proTx.sig = operatorKey.Sign(::SerializeHash(proTx)); SetTxPayload(tx, proTx); - SignTransaction(tx, coinbaseKey); + SignTransaction(mempool, tx, coinbaseKey); return tx; } -static CMutableTransaction CreateProUpRegTx(SimpleUTXOMap& utxos, const uint256& proTxHash, const CKey& mnKey, const CBLSPublicKey& pubKeyOperator, const CKeyID& keyIDVoting, const CScript& scriptPayout, const CKey& coinbaseKey) +static CMutableTransaction CreateProUpRegTx(const CTxMemPool& mempool, SimpleUTXOMap& utxos, const uint256& proTxHash, const CKey& mnKey, const CBLSPublicKey& pubKeyOperator, const CKeyID& keyIDVoting, const CScript& scriptPayout, const CKey& coinbaseKey) { CProUpRegTx proTx; proTx.proTxHash = proTxHash; @@ -151,12 +151,12 @@ static CMutableTransaction CreateProUpRegTx(SimpleUTXOMap& utxos, const uint256& proTx.inputsHash = CalcTxInputsHash(CTransaction(tx)); CHashSigner::SignHash(::SerializeHash(proTx), mnKey, proTx.vchSig); SetTxPayload(tx, proTx); - SignTransaction(tx, coinbaseKey); + SignTransaction(mempool, tx, coinbaseKey); return tx; } -static CMutableTransaction CreateProUpRevTx(SimpleUTXOMap& utxos, const uint256& proTxHash, const CBLSSecretKey& operatorKey, const CKey& coinbaseKey) +static CMutableTransaction CreateProUpRevTx(const CTxMemPool& mempool, SimpleUTXOMap& utxos, const uint256& proTxHash, const CBLSSecretKey& operatorKey, const CKey& coinbaseKey) { CProUpRevTx proTx; proTx.proTxHash = proTxHash; @@ -168,7 +168,7 @@ static CMutableTransaction CreateProUpRevTx(SimpleUTXOMap& utxos, const uint256& proTx.inputsHash = CalcTxInputsHash(CTransaction(tx)); proTx.sig = operatorKey.Sign(::SerializeHash(proTx)); SetTxPayload(tx, proTx); - SignTransaction(tx, coinbaseKey); + SignTransaction(mempool, tx, coinbaseKey); return tx; } @@ -214,13 +214,13 @@ static CDeterministicMNCPtr FindPayoutDmn(const CBlock& block) return nullptr; } -static bool CheckTransactionSignature(const CMutableTransaction& tx) +static bool CheckTransactionSignature(const CTxMemPool& mempool, const CMutableTransaction& tx) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const auto& txin = tx.vin[i]; - CTransactionRef txFrom; uint256 hashBlock; - BOOST_ASSERT(GetTransaction(txin.prevout.hash, txFrom, Params().GetConsensus(), hashBlock)); + CTransactionRef txFrom = GetTransaction(/* block_index */ nullptr, &mempool, txin.prevout.hash, Params().GetConsensus(), hashBlock); + BOOST_ASSERT(txFrom); CAmount amount = txFrom->vout[txin.prevout.n].nValue; if (!VerifyScript(txin.scriptSig, txFrom->vout[txin.prevout.n].scriptPubKey, STANDARD_SCRIPT_VERIFY_FLAGS, MutableTransactionSignatureChecker(&tx, i, amount))) { @@ -238,7 +238,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_activation, TestChainDIP3BeforeActivationSetup) CKey ownerKey; CBLSSecretKey operatorKey; CTxDestination payoutDest = DecodeDestination("yRq1Ky1AfFmf597rnotj7QRxsDUKePVWNF"); - auto tx = CreateProRegTx(utxos, 1, GetScriptForDestination(payoutDest), coinbaseKey, ownerKey, operatorKey); + auto tx = CreateProRegTx(*m_node.mempool, utxos, 1, GetScriptForDestination(payoutDest), coinbaseKey, ownerKey, operatorKey); std::vector txns = {tx}; int nHeight = ::ChainActive().Height(); @@ -282,7 +282,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) for (size_t i = 0; i < 6; i++) { CKey ownerKey; CBLSSecretKey operatorKey; - auto tx = CreateProRegTx(utxos, port++, GenerateRandomAddress(), coinbaseKey, ownerKey, operatorKey); + auto tx = CreateProRegTx(*m_node.mempool, utxos, port++, GenerateRandomAddress(), coinbaseKey, ownerKey, operatorKey); dmnHashes.emplace_back(tx.GetHash()); ownerKeys.emplace(tx.GetHash(), ownerKey); operatorKeys.emplace(tx.GetHash(), operatorKey); @@ -300,8 +300,8 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) BOOST_ASSERT(CheckProRegTx(CTransaction(tx2), ::ChainActive().Tip(), dummyState, ::ChainstateActive().CoinsTip(), true)); } // But the signature should not verify anymore - BOOST_ASSERT(CheckTransactionSignature(tx)); - BOOST_ASSERT(!CheckTransactionSignature(tx2)); + BOOST_ASSERT(CheckTransactionSignature(*m_node.mempool, tx)); + BOOST_ASSERT(!CheckTransactionSignature(*m_node.mempool, tx2)); CreateAndProcessBlock({tx}, coinbaseKey); deterministicMNManager->UpdatedBlockTip(::ChainActive().Tip()); @@ -339,7 +339,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) for (size_t j = 0; j < 3; j++) { CKey ownerKey; CBLSSecretKey operatorKey; - auto tx = CreateProRegTx(utxos, port++, GenerateRandomAddress(), coinbaseKey, ownerKey, operatorKey); + auto tx = CreateProRegTx(*m_node.mempool, utxos, port++, GenerateRandomAddress(), coinbaseKey, ownerKey, operatorKey); dmnHashes.emplace_back(tx.GetHash()); ownerKeys.emplace(tx.GetHash(), ownerKey); operatorKeys.emplace(tx.GetHash(), operatorKey); @@ -357,7 +357,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) } // test ProUpServTx - auto tx = CreateProUpServTx(utxos, dmnHashes[0], operatorKeys[dmnHashes[0]], 1000, CScript(), coinbaseKey); + auto tx = CreateProUpServTx(*m_node.mempool, utxos, dmnHashes[0], operatorKeys[dmnHashes[0]], 1000, CScript(), coinbaseKey); CreateAndProcessBlock({tx}, coinbaseKey); deterministicMNManager->UpdatedBlockTip(::ChainActive().Tip()); BOOST_CHECK_EQUAL(::ChainActive().Height(), nHeight + 1); @@ -367,7 +367,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) BOOST_ASSERT(dmn != nullptr && dmn->pdmnState->addr.GetPort() == 1000); // test ProUpRevTx - tx = CreateProUpRevTx(utxos, dmnHashes[0], operatorKeys[dmnHashes[0]], coinbaseKey); + tx = CreateProUpRevTx(*m_node.mempool, utxos, dmnHashes[0], operatorKeys[dmnHashes[0]], coinbaseKey); CreateAndProcessBlock({tx}, coinbaseKey); deterministicMNManager->UpdatedBlockTip(::ChainActive().Tip()); BOOST_CHECK_EQUAL(::ChainActive().Height(), nHeight + 1); @@ -396,7 +396,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) CBLSSecretKey newOperatorKey; newOperatorKey.MakeNewKey(); dmn = deterministicMNManager->GetListAtChainTip().GetMN(dmnHashes[0]); - tx = CreateProUpRegTx(utxos, dmnHashes[0], ownerKeys[dmnHashes[0]], newOperatorKey.GetPublicKey(), ownerKeys[dmnHashes[0]].GetPubKey().GetID(), dmn->pdmnState->scriptPayout, coinbaseKey); + tx = CreateProUpRegTx(*m_node.mempool, utxos, dmnHashes[0], ownerKeys[dmnHashes[0]], newOperatorKey.GetPublicKey(), ownerKeys[dmnHashes[0]].GetPubKey().GetID(), dmn->pdmnState->scriptPayout, coinbaseKey); // check malleability protection again, but this time by also relying on the signature inside the ProUpRegTx auto tx2 = MalleateProTxPayout(tx); CValidationState dummyState; @@ -405,15 +405,15 @@ BOOST_FIXTURE_TEST_CASE(dip3_protx, TestChainDIP3Setup) BOOST_ASSERT(CheckProUpRegTx(CTransaction(tx), ::ChainActive().Tip(), dummyState, ::ChainstateActive().CoinsTip(), true)); BOOST_ASSERT(!CheckProUpRegTx(CTransaction(tx2), ::ChainActive().Tip(), dummyState, ::ChainstateActive().CoinsTip(), true)); } - BOOST_ASSERT(CheckTransactionSignature(tx)); - BOOST_ASSERT(!CheckTransactionSignature(tx2)); + BOOST_ASSERT(CheckTransactionSignature(*m_node.mempool, tx)); + BOOST_ASSERT(!CheckTransactionSignature(*m_node.mempool, tx2)); // now process the block CreateAndProcessBlock({tx}, coinbaseKey); deterministicMNManager->UpdatedBlockTip(::ChainActive().Tip()); BOOST_CHECK_EQUAL(::ChainActive().Height(), nHeight + 1); nHeight++; - tx = CreateProUpServTx(utxos, dmnHashes[0], newOperatorKey, 100, CScript(), coinbaseKey); + tx = CreateProUpServTx(*m_node.mempool, utxos, dmnHashes[0], newOperatorKey, 100, CScript(), coinbaseKey); CreateAndProcessBlock({tx}, coinbaseKey); deterministicMNManager->UpdatedBlockTip(::ChainActive().Tip()); BOOST_CHECK_EQUAL(::ChainActive().Height(), nHeight + 1); @@ -467,7 +467,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_test_mempool_reorg, TestChainDIP3Setup) // Create a MN with an external collateral CMutableTransaction tx_collateral; FundTransaction(tx_collateral, utxos, scriptCollateral, 1000 * COIN, coinbaseKey); - SignTransaction(tx_collateral, coinbaseKey); + SignTransaction(*m_node.mempool, tx_collateral, coinbaseKey); auto block = std::make_shared(CreateBlock({tx_collateral}, coinbaseKey)); BOOST_ASSERT(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); @@ -496,14 +496,14 @@ BOOST_FIXTURE_TEST_CASE(dip3_test_mempool_reorg, TestChainDIP3Setup) payload.inputsHash = CalcTxInputsHash(CTransaction(tx_reg)); CMessageSigner::SignMessage(payload.MakeSignString(), payload.vchSig, collateralKey); SetTxPayload(tx_reg, payload); - SignTransaction(tx_reg, coinbaseKey); + SignTransaction(*m_node.mempool, tx_reg, coinbaseKey); CTxMemPool testPool; TestMemPoolEntryHelper entry; LOCK2(cs_main, testPool.cs); // Create ProUpServ and test block reorg which double-spend ProRegTx - auto tx_up_serv = CreateProUpServTx(utxos, tx_reg.GetHash(), operatorKey, 2, CScript(), coinbaseKey); + auto tx_up_serv = CreateProUpServTx(*m_node.mempool, utxos, tx_reg.GetHash(), operatorKey, 2, CScript(), coinbaseKey); testPool.addUnchecked(entry.FromTx(tx_up_serv)); // A disconnected block would insert ProRegTx back into mempool testPool.addUnchecked(entry.FromTx(tx_reg)); @@ -513,7 +513,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_test_mempool_reorg, TestChainDIP3Setup) CMutableTransaction tx_reg_ds; tx_reg_ds.vin = tx_reg.vin; tx_reg_ds.vout.emplace_back(0, CScript() << OP_RETURN); - SignTransaction(tx_reg_ds, coinbaseKey); + SignTransaction(*m_node.mempool, tx_reg_ds, coinbaseKey); // Check mempool as if a new block with tx_reg_ds was connected instead of the old one with tx_reg std::vector block_reorg; @@ -529,7 +529,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_test_mempool_dual_proregtx, TestChainDIP3Setup) // Create a MN CKey ownerKey1; CBLSSecretKey operatorKey1; - auto tx_reg1 = CreateProRegTx(utxos, 1, GenerateRandomAddress(), coinbaseKey, ownerKey1, operatorKey1); + auto tx_reg1 = CreateProRegTx(*m_node.mempool, utxos, 1, GenerateRandomAddress(), coinbaseKey, ownerKey1, operatorKey1); // Create a MN with an external collateral that references tx_reg1 CKey ownerKey; @@ -565,7 +565,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_test_mempool_dual_proregtx, TestChainDIP3Setup) payload.inputsHash = CalcTxInputsHash(CTransaction(tx_reg2)); CMessageSigner::SignMessage(payload.MakeSignString(), payload.vchSig, collateralKey); SetTxPayload(tx_reg2, payload); - SignTransaction(tx_reg2, coinbaseKey); + SignTransaction(*m_node.mempool, tx_reg2, coinbaseKey); CTxMemPool testPool; TestMemPoolEntryHelper entry; @@ -597,7 +597,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_verify_db, TestChainDIP3Setup) // Create a MN with an external collateral CMutableTransaction tx_collateral; FundTransaction(tx_collateral, utxos, scriptCollateral, 1000 * COIN, coinbaseKey); - SignTransaction(tx_collateral, coinbaseKey); + SignTransaction(*m_node.mempool, tx_collateral, coinbaseKey); auto block = std::make_shared(CreateBlock({tx_collateral}, coinbaseKey)); BOOST_ASSERT(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); @@ -626,7 +626,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_verify_db, TestChainDIP3Setup) payload.inputsHash = CalcTxInputsHash(CTransaction(tx_reg)); CMessageSigner::SignMessage(payload.MakeSignString(), payload.vchSig, collateralKey); SetTxPayload(tx_reg, payload); - SignTransaction(tx_reg, coinbaseKey); + SignTransaction(*m_node.mempool, tx_reg, coinbaseKey); auto tx_reg_hash = tx_reg.GetHash(); @@ -640,7 +640,7 @@ BOOST_FIXTURE_TEST_CASE(dip3_verify_db, TestChainDIP3Setup) // Now spend the collateral while updating the same MN SimpleUTXOMap collateral_utxos; collateral_utxos.emplace(payload.collateralOutpoint, std::make_pair(1, 1000)); - auto proUpRevTx = CreateProUpRevTx(collateral_utxos, tx_reg_hash, operatorKey, collateralKey); + auto proUpRevTx = CreateProUpRevTx(*m_node.mempool, collateral_utxos, tx_reg_hash, operatorKey, collateralKey); block = std::make_shared(CreateBlock({proUpRevTx}, coinbaseKey)); BOOST_ASSERT(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 6616ea1be054..6f08647e83c1 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -70,12 +70,8 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - // txindex job may be scheduled, so stop scheduler before destructing - m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() + // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed + SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index f264ade4c59b..378424d98fea 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -143,8 +143,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha g_txindex = MakeUnique(1 << 20, true); g_txindex->Start(); deterministicMNManager.reset(new CDeterministicMNManager(*evoDb, *m_node.connman)); - llmq::InitLLMQSystem(*evoDb, *m_node.connman, true); - ::ChainstateActive().InitCoinsCache(); + llmq::InitLLMQSystem(*evoDb, *m_node.mempool, *m_node.connman, true); + ::ChainstateActive().InitCoinsCache(1 << 23); assert(::ChainstateActive().CanFlushToDisk()); if (!LoadGenesisBlock(chainparams)) { throw std::runtime_error("LoadGenesisBlock failed."); @@ -177,9 +177,9 @@ TestingSetup::~TestingSetup() GetMainSignals().UnregisterBackgroundSignalScheduler(); m_node.connman.reset(); m_node.banman.reset(); + UnloadBlockIndex(m_node.mempool); m_node.mempool = nullptr; m_node.scheduler.reset(); - UnloadBlockIndex(); llmq::DestroyLLMQSystem(); m_node.chainman->Reset(); m_node.chainman = nullptr; diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp new file mode 100644 index 000000000000..5cd2a60e6f84 --- /dev/null +++ b/src/test/validation_chainstate_tests.cpp @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) + +//! Test resizing coins-related CChainState caches during runtime. +//! +BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) +{ + // TODO: this is a hack to avoid txindex crash on BlockConnected trigered by ActivateBestChain + g_txindex->Interrupt(); + g_txindex->Stop(); + + ChainstateManager manager; + + //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. + auto add_coin = [](CCoinsViewCache& coins_view) -> COutPoint { + Coin newcoin; + uint256 txid = InsecureRand256(); + COutPoint outp{txid, 0}; + newcoin.nHeight = 1; + newcoin.out.nValue = InsecureRand32(); + newcoin.out.scriptPubKey.assign((uint32_t)56, 1); + coins_view.AddCoin(outp, std::move(newcoin), false); + + return outp; + }; + + CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate()); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); + + // Add a coin to the in-memory cache, upsize once, then downsize. + { + LOCK(::cs_main); + auto outpoint = add_coin(c1.CoinsTip()); + + // Set a meaningless bestblock value in the coinsview cache - otherwise we won't + // flush during ResizecoinsCaches() and will subsequently hit an assertion. + c1.CoinsTip().SetBestBlock(InsecureRand256()); + + BOOST_CHECK(c1.CoinsTip().HaveCoinInCache(outpoint)); + + c1.ResizeCoinsCaches( + 1 << 24, // upsizing the coinsview cache + 1 << 22 // downsizing the coinsdb cache + ); + + // View should still have the coin cached, since we haven't destructed the cache on upsize. + BOOST_CHECK(c1.CoinsTip().HaveCoinInCache(outpoint)); + + c1.ResizeCoinsCaches( + 1 << 22, // downsizing the coinsview cache + 1 << 23 // upsizing the coinsdb cache + ); + + // The view cache should be empty since we had to destruct to downsize. + BOOST_CHECK(!c1.CoinsTip().HaveCoinInCache(outpoint)); + } + + // Avoid triggering the address sanitizer. + WITH_LOCK(::cs_main, manager.Unload()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index c43123a0a765..2990d5dccdf3 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -3,13 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include -#include #include -#include #include +#include #include #include +#include #include +#include #include @@ -32,13 +33,11 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a legacy (IBD) chainstate. // - ENTER_CRITICAL_SECTION(cs_main); - CChainState& c1 = manager.InitializeChainstate(); - LEAVE_CRITICAL_SECTION(cs_main); + CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate()); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); - WITH_LOCK(::cs_main, c1.InitCoinsCache()); + WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); @@ -60,13 +59,11 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // - ENTER_CRITICAL_SECTION(cs_main); - CChainState& c2 = manager.InitializeChainstate(GetRandHash()); - LEAVE_CRITICAL_SECTION(cs_main); + CChainState& c2 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); - WITH_LOCK(::cs_main, c2.InitCoinsCache()); + WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); // Unlike c1, which doesn't have any blocks. Gets us different tip, height. c2.LoadGenesisBlock(chainparams); CValidationState _; @@ -102,8 +99,64 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(validated_tip, exp_tip); - // Avoid triggering the address sanitizer. + // Let scheduler events finish running to avoid accessing memory that is going to be unloaded + SyncWithValidationInterfaceQueue(); + WITH_LOCK(::cs_main, manager.Unload()); } +//! Test rebalancing the caches associated with each chainstate. +BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) +{ + // TODO: this is a hack to avoid txindex crash on BlockConnected trigered by ActivateBestChain + g_txindex->Interrupt(); + g_txindex->Stop(); + + ChainstateManager manager; + size_t max_cache = 10000; + manager.m_total_coinsdb_cache = max_cache; + manager.m_total_coinstip_cache = max_cache; + + std::vector chainstates; + + // Create a legacy (IBD) chainstate. + // + CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate()); + chainstates.push_back(&c1); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + + { + LOCK(::cs_main); + c1.InitCoinsCache(1 << 23); + c1.CoinsTip().SetBestBlock(InsecureRand256()); + manager.MaybeRebalanceCaches(); + } + + BOOST_CHECK_EQUAL(c1.m_coinstip_cache_size_bytes, max_cache); + BOOST_CHECK_EQUAL(c1.m_coinsdb_cache_size_bytes, max_cache); + + // Create a snapshot-based chainstate. + // + CChainState& c2 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(GetRandHash())); + chainstates.push_back(&c2); + c2.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + + { + LOCK(::cs_main); + c2.InitCoinsCache(1 << 23); + c2.CoinsTip().SetBestBlock(InsecureRand256()); + manager.MaybeRebalanceCaches(); + } + + // Since both chainstates are considered to be in initial block download, + // the snapshot chainstate should take priority. + BOOST_CHECK_CLOSE(c1.m_coinstip_cache_size_bytes, max_cache * 0.05, 1); + BOOST_CHECK_CLOSE(c1.m_coinsdb_cache_size_bytes, max_cache * 0.05, 1); + BOOST_CHECK_CLOSE(c2.m_coinstip_cache_size_bytes, max_cache * 0.95, 1); + BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); + +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index c24164528f44..fbba898cfac6 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) BlockManager blockman{}; CChainState chainstate{blockman}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); - WITH_LOCK(::cs_main, chainstate.InitCoinsCache()); + WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); CTxMemPool tx_pool{}; constexpr bool is_64_bit = sizeof(void*) == 8; @@ -56,7 +56,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // Without any coins in the cache, we shouldn't need to flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); // If the initial memory allocations of cacheCoins don't match these common @@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); @@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); } @@ -100,26 +100,26 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 4; ++i) { add_coin(view); print_view_mem_usage(view); - if (chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == + if (chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == CoinsCacheSizeState::CRITICAL) { break; } } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); // Passing non-zero max mempool usage should allow us more headroom. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); for (int i{0}; i < 3; ++i) { add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); } @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) BOOST_CHECK(usage_percentage >= 0.9); BOOST_CHECK(usage_percentage < 1); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), CoinsCacheSizeState::LARGE); } @@ -143,7 +143,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 1000; ++i) { add_coin(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool), + chainstate.GetCoinsCacheSizeState(&tx_pool), CoinsCacheSizeState::OK); } @@ -151,7 +151,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // preallocated memory that doesn't get reclaimed even after flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); view.SetBestBlock(InsecureRand256()); @@ -159,7 +159,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); } diff --git a/src/txdb.cpp b/src/txdb.cpp index d63ad71ada75..462c89db1b77 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -45,35 +46,45 @@ struct CoinEntry { } -CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : db(ldb_path, nCacheSize, fMemory, fWipe, true) +CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : + m_db(MakeUnique(ldb_path, nCacheSize, fMemory, fWipe, true)), + m_ldb_path(ldb_path), + m_is_memory(fMemory) { } + +void CCoinsViewDB::ResizeCache(size_t new_cache_size) { + // Have to do a reset first to get the original `m_db` state to release its + // filesystem lock. + m_db.reset(); + m_db = MakeUnique( + m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); } bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { - return db.Read(CoinEntry(&outpoint), coin); + return m_db->Read(CoinEntry(&outpoint), coin); } bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { - return db.Exists(CoinEntry(&outpoint)); + return m_db->Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { uint256 hashBestChain; - if (!db.Read(DB_BEST_BLOCK, hashBestChain)) + if (!m_db->Read(DB_BEST_BLOCK, hashBestChain)) return uint256(); return hashBestChain; } std::vector CCoinsViewDB::GetHeadBlocks() const { std::vector vhashHeadBlocks; - if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) { + if (!m_db->Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) { return std::vector(); } return vhashHeadBlocks; } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CDBBatch batch(db); + CDBBatch batch(*m_db); size_t count = 0; size_t changed = 0; size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize); @@ -111,7 +122,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { mapCoins.erase(itOld); if (batch.SizeEstimate() > batch_size) { LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); - db.WriteBatch(batch); + m_db->WriteBatch(batch); batch.Clear(); if (crash_simulate) { static FastRandomContext rng; @@ -128,14 +139,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { batch.Write(DB_BEST_BLOCK, hashBlock); LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); - bool ret = db.WriteBatch(batch); + bool ret = m_db->WriteBatch(batch); LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return ret; } size_t CCoinsViewDB::EstimateSize() const { - return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); + return m_db->EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -162,7 +173,7 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { CCoinsViewCursor *CCoinsViewDB::Cursor() const { - CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(db).NewIterator(), GetBestBlock()); + CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(*m_db).NewIterator(), GetBestBlock()); /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ @@ -470,7 +481,7 @@ class CCoins * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. */ bool CCoinsViewDB::Upgrade() { - std::unique_ptr pcursor(db.NewIterator()); + std::unique_ptr pcursor(m_db->NewIterator()); pcursor->Seek(std::make_pair(DB_COINS, uint256())); if (!pcursor->Valid()) { return true; @@ -481,7 +492,7 @@ bool CCoinsViewDB::Upgrade() { LogPrintf("[0%%]..."); /* Continued */ uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true); size_t batch_size = 1 << 24; - CDBBatch batch(db); + CDBBatch batch(*m_db); int reportDone = 0; std::pair key; std::pair prev_key = {DB_COINS, uint256()}; @@ -516,9 +527,9 @@ bool CCoinsViewDB::Upgrade() { } batch.Erase(key); if (batch.SizeEstimate() > batch_size) { - db.WriteBatch(batch); + m_db->WriteBatch(batch); batch.Clear(); - db.CompactRange(prev_key, key); + m_db->CompactRange(prev_key, key); prev_key = key; } pcursor->Next(); @@ -526,8 +537,8 @@ bool CCoinsViewDB::Upgrade() { break; } } - db.WriteBatch(batch); - db.CompactRange({DB_COINS, uint256()}, key); + m_db->WriteBatch(batch); + m_db->CompactRange({DB_COINS, uint256()}, key); uiInterface.ShowProgress("", 100, false); LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); diff --git a/src/txdb.h b/src/txdb.h index 0ea5f15044c9..126f1ef23331 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -40,11 +40,16 @@ static const int64_t max_filter_index_cache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; +// Actually declared in validation.cpp; can't include because of circular dependency. +extern RecursiveMutex cs_main; + /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { protected: - CDBWrapper db; + std::unique_ptr m_db; + fs::path m_ldb_path; + bool m_is_memory; public: /** * @param[in] ldb_path Location in the filesystem where leveldb data will be stored. @@ -62,6 +67,9 @@ class CCoinsViewDB final : public CCoinsView //! Attempt to update from an older database format. Returns whether an error occurred. bool Upgrade(); size_t EstimateSize() const override; + + //! Dynamically alter the underlying leveldb cache size. + void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ diff --git a/src/validation.cpp b/src/validation.cpp index 6ac736bab72d..8639a409a09b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -141,7 +141,6 @@ bool fPruneMode = false; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -size_t nCoinCacheUsage = 5000 * 300; uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; @@ -567,9 +566,9 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool llmq::CInstantSendLockPtr conflictLock = llmq::quorumInstantSendManager->GetConflictingLock(tx); if (conflictLock) { - CTransactionRef txConflict; uint256 hashBlock; - if (GetTransaction(conflictLock->txid, txConflict, chainparams.GetConsensus(), hashBlock)) { + CTransactionRef txConflict = GetTransaction(/* block_index */ nullptr, &pool, conflictLock->txid, chainparams.GetConsensus(), hashBlock); + if (txConflict) { GetMainSignals().NotifyInstantSendDoubleSpendAttempt(ptx, txConflict); } return state.DoS(10, error("AcceptToMemoryPool : Transaction %s conflicts with locked TX %s", @@ -891,45 +890,33 @@ bool GetAddressUnspent(uint160 addressHash, int type, return true; } -/** - * Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock. - * If blockIndex is provided, the transaction is fetched from the corresponding block. - */ -bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus::Params& consensusParams, uint256& hashBlock, const CBlockIndex* const block_index) +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) { LOCK(cs_main); - if (!block_index) { - CTransactionRef ptx = mempool.get(hash); - if (ptx) { - txOut = ptx; - return true; - } - - if (g_txindex) { - return g_txindex->FindTx(hash, hashBlock, txOut); - } - } else { + if (block_index) { CBlock block; if (ReadBlockFromDisk(block, block_index, consensusParams)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { - txOut = tx; hashBlock = block_index->GetBlockHash(); - return true; + return tx; } } } + return nullptr; } - - return false; + if (mempool) { + CTransactionRef ptx = mempool->get(hash); + if (ptx) return ptx; + } + if (g_txindex) { + CTransactionRef tx; + if (g_txindex->FindTx(hash, hashBlock, tx)) return tx; + } + return nullptr; } - - - - - ////////////////////////////////////////////////////////////////////////////// // // CBlock and CBlockIndex @@ -1163,9 +1150,10 @@ void CChainState::InitCoinsDB( leveldb_name, cache_size_bytes, in_memory, should_wipe); } -void CChainState::InitCoinsCache() +void CChainState::InitCoinsCache(size_t cache_size_bytes) { assert(m_coins_views != nullptr); + m_coinstip_cache_size_bytes = cache_size_bytes; m_coins_views->InitCache(); } @@ -2414,20 +2402,20 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl return true; } -CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_pool) +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool* tx_pool) { return this->GetCoinsCacheSizeState( tx_pool, - nCoinCacheUsage, + m_coinstip_cache_size_bytes, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( - const CTxMemPool& tx_pool, + const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) { - int64_t nMempoolUsage = tx_pool.DynamicMemoryUsage(); + const int64_t nMempoolUsage = tx_pool ? tx_pool->DynamicMemoryUsage() : 0; int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); int64_t nTotalSpace = max_coins_cache_size_bytes + std::max(max_mempool_size_bytes - nMempoolUsage, 0); @@ -2468,7 +2456,7 @@ bool CChainState::FlushStateToDisk( { bool fFlushForPrune = false; bool fDoFullFlush = false; - CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(::mempool); + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&::mempool); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { @@ -4684,7 +4672,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { + if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= ::ChainstateActive().m_coinstip_cache_size_bytes) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { @@ -4844,13 +4832,13 @@ void CChainState::UnloadBlockIndex() { // May NOT be used after any connections are up as much // of the peer-processing logic assumes a consistent // block index state -void UnloadBlockIndex() +void UnloadBlockIndex(CTxMemPool* mempool) { LOCK(cs_main); g_chainman.Unload(); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; - mempool.clear(); + if (mempool) mempool->clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; setDirtyBlockIndex.clear(); @@ -5264,6 +5252,39 @@ std::string CChainState::ToString() tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); } +bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) +{ + if (coinstip_size == m_coinstip_cache_size_bytes && + coinsdb_size == m_coinsdb_cache_size_bytes) { + // Cache sizes are unchanged, no need to continue. + return true; + } + size_t old_coinstip_size = m_coinstip_cache_size_bytes; + m_coinstip_cache_size_bytes = coinstip_size; + m_coinsdb_cache_size_bytes = coinsdb_size; + CoinsDB().ResizeCache(coinsdb_size); + + LogPrintf("[%s] resized coinsdb cache to %.1f MiB\n", + this->ToString(), coinsdb_size * (1.0 / 1024 / 1024)); + LogPrintf("[%s] resized coinstip cache to %.1f MiB\n", + this->ToString(), coinstip_size * (1.0 / 1024 / 1024)); + + CValidationState state; + const CChainParams& chainparams = Params(); + + bool ret; + + if (coinstip_size > old_coinstip_size) { + // Likely no need to flush if cache sizes have grown. + ret = FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED); + } else { + // Otherwise, flush state to disk and deallocate the in-memory coins map. + ret = FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS); + CoinsTip().ReallocateCache(); + } + return ret; +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); } @@ -5522,10 +5543,10 @@ CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blo return *to_modify; } -CChain& ChainstateManager::ActiveChain() const +CChainState& ChainstateManager::ActiveChainstate() const { assert(m_active_chainstate); - return m_active_chainstate->m_chain; + return *m_active_chainstate; } bool ChainstateManager::IsSnapshotActive() const @@ -5564,3 +5585,33 @@ void ChainstateManager::Reset() m_active_chainstate = nullptr; m_snapshot_validated = false; } + +void ChainstateManager::MaybeRebalanceCaches() +{ + if (m_ibd_chainstate && !m_snapshot_chainstate) { + LogPrintf("[snapshot] allocating all cache to the IBD chainstate\n"); + // Allocate everything to the IBD chainstate. + m_ibd_chainstate->ResizeCoinsCaches(m_total_coinstip_cache, m_total_coinsdb_cache); + } + else if (m_snapshot_chainstate && !m_ibd_chainstate) { + LogPrintf("[snapshot] allocating all cache to the snapshot chainstate\n"); + // Allocate everything to the snapshot chainstate. + m_snapshot_chainstate->ResizeCoinsCaches(m_total_coinstip_cache, m_total_coinsdb_cache); + } + else if (m_ibd_chainstate && m_snapshot_chainstate) { + // If both chainstates exist, determine who needs more cache based on IBD status. + // + // Note: shrink caches first so that we don't inadvertently overwhelm available memory. + if (m_snapshot_chainstate->IsInitialBlockDownload()) { + m_ibd_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.05, m_total_coinsdb_cache * 0.05); + m_snapshot_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.95, m_total_coinsdb_cache * 0.95); + } else { + m_snapshot_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.05, m_total_coinsdb_cache * 0.05); + m_ibd_chainstate->ResizeCoinsCaches( + m_total_coinstip_cache * 0.95, m_total_coinsdb_cache * 0.95); + } + } +} diff --git a/src/validation.h b/src/validation.h index 21c6f0d16202..0da950c5e262 100644 --- a/src/validation.h +++ b/src/validation.h @@ -23,6 +23,7 @@ #include #include // For CTxMemPool::cs #include +#include #include #include @@ -141,7 +142,6 @@ extern bool g_parallel_script_checks; extern bool fRequireStandard; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; -extern size_t nCoinCacheUsage; /** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */ extern CFeeRate minRelayTxFee; /** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ @@ -178,13 +178,24 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Unload database information */ -void UnloadBlockIndex(); +void UnloadBlockIndex(CTxMemPool* mempool); /** Run instances of script checking worker threads */ void StartScriptCheckWorkerThreads(int threads_num); /** Stop all of the script checking worker threads */ void StopScriptCheckWorkerThreads(); -/** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::Params& params, uint256& hashBlock, const CBlockIndex* const blockIndex = nullptr); +/** + * Return transaction from the block at block_index. + * If block_index is not provided, fall back to mempool. + * If mempool is not provided or the tx couldn't be found in mempool, fall back to g_txindex. + * + * @param[in] block_index The block to read from disk, or nullptr + * @param[in] mempool If block_index is not provided, look in the mempool, if provided + * @param[in] hash The txid + * @param[in] consensusParams The params + * @param[out] hashBlock The hash of block_index, if the tx was found via block_index + * @returns The tx if found, otherwise nullptr + */ +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); /** * Find the best known block, and make it the tip of the block chain * @@ -537,7 +548,7 @@ class CChainState //! Initialize the in-memory coins cache (to be done after the health of the on-disk database //! is verified). - void InitCoinsCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void InitCoinsCache(size_t cache_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! @returns whether or not the CoinsViews object has been fully initialized and we can //! safely flush this object to disk. @@ -586,6 +597,17 @@ class CChainState //! Destructs all objects related to accessing the UTXO set. void ResetCoinsViews() { m_coins_views.reset(); } + //! The cache size of the on-disk coins view. + size_t m_coinsdb_cache_size_bytes{0}; + + //! The cache size of the in-memory coins view. + size_t m_coinstip_cache_size_bytes{0}; + + //! Resize the CoinsViews caches dynamically and flush state to disk. + //! @returns true unless an error occurred during the flush. + bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -669,11 +691,11 @@ class CChainState //! Dictates whether we need to flush the cache to disk or not. //! //! @return the state of the size of the coins cache. - CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool& tx_pool) + CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool* tx_pool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); CoinsCacheSizeState GetCoinsCacheSizeState( - const CTxMemPool& tx_pool, + const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -809,6 +831,14 @@ class ChainstateManager //! chainstate to avoid duplicating block metadata. BlockManager m_blockman GUARDED_BY(::cs_main); + //! The total number of bytes available for us to use across all in-memory + //! coins caches. This will be split somehow across chainstates. + int64_t m_total_coinstip_cache{0}; + // + //! The total number of bytes available for us to use across all leveldb + //! coins databases. This will be split somehow across chainstates. + int64_t m_total_coinsdb_cache{0}; + //! Instantiate a new chainstate and assign it based upon whether it is //! from a snapshot. //! @@ -821,7 +851,8 @@ class ChainstateManager std::vector GetAll(); //! The most-work chain. - CChain& ActiveChain() const; + CChainState& ActiveChainstate() const; + CChain& ActiveChain() const { return ActiveChainstate().m_chain; } int ActiveHeight() const { return ActiveChain().Height(); } CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } @@ -897,18 +928,22 @@ class ChainstateManager //! Clear (deconstruct) chainstate data. void Reset(); + + //! Check to see if caches are out of balance and if so, call + //! ResizeCoinsCaches() as needed. + void MaybeRebalanceCaches() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; /** DEPRECATED! Please use node.chainman instead. May only be used in validation.cpp internally */ extern ChainstateManager g_chainman GUARDED_BY(::cs_main); -/** @returns the most-work chain. */ -CChain& ChainActive(); - -/** @returns the most-work valid chainstate. */ +/** Please prefer the identical ChainstateManager::ActiveChainstate */ CChainState& ChainstateActive(); -/** @returns the global block index map. */ +/** Please prefer the identical ChainstateManager::ActiveChain */ +CChain& ChainActive(); + +/** Please prefer the identical ChainstateManager::BlockIndex */ BlockMap& BlockIndex(); /** @returns the global previous block index map. */ diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 index 000000000000..0f12ba46312f --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / self.chain / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '65a627bab5f50aea8e69acbf9fcbe6e5162bd556ce1c46f0ec4452afaedbb702') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, '1d34e230e9d7d5691aaa03684492798ca8f92a9254c6cf80528241bf35939869') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index cace9cb12911..0c02398afd97 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -217,6 +217,7 @@ 'rpc_uptime.py', 'wallet_resendwallettransactions.py', 'wallet_fallbackfee.py', + 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'p2p_unrequested_blocks.py', # NOTE: needs dash_hash to pass 'feature_shutdown.py',