merge bitcoin#16240...21526: assumeutxo project backports (part 2)

contrib/devtools/utxo_snapshot.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+export LC_ALL=C
+
+set -ueo pipefail
+
+if (( $# < 3 )); then
+  echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <dash-cli-call ...>'
+  echo
+  echo "  if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the "
+  echo "  expected assumeutxo hash"
+  echo
+  echo 'Examples:'
+  echo
+  echo "  ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/dash-cli -datadir=\$(pwd)/testdata"
+  echo '  ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/dash-cli'
+  exit 1
+fi
+
+GENERATE_AT_HEIGHT="${1}"; shift;
+OUTPUT_PATH="${1}"; shift;
+# Most of the calls we make take a while to run, so pad with a lengthy timeout.
+BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999"
+
+# Block we'll invalidate/reconsider to rewind/fast-forward the chain.
+PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) )
+
+(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while")
+${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
+
+if [[ "${OUTPUT_PATH}" = "-" ]]; then
+  (>&2 echo "Generating txoutset info...")
+  ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g'
+else
+  (>&2 echo "Generating UTXO snapshot...")
+  ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"
+fi
+
+(>&2 echo "Restoring chain to original height; this may take a while")
+${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}"

src/Makefile.am

@@ -238,6 +238,7 @@ BITCOIN_CORE_H = \
   node/coinstats.h \
   node/context.h \
   node/transaction.h \
+  node/utxo_snapshot.h \
   noui.h \
   optional.h \
   policy/feerate.h \

src/Makefile.test.include

@@ -229,6 +229,7 @@ BITCOIN_TESTS =\
   test/uint256_tests.cpp \
   test/util_tests.cpp \
   test/validation_block_tests.cpp \
+  test/validation_chainstate_tests.cpp \
   test/validation_chainstatemanager_tests.cpp \
   test/validation_flush_tests.cpp \
   test/versionbits_tests.cpp

src/coins.cpp

@@ -257,6 +257,14 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const
     return true;
 }
 
+void CCoinsViewCache::ReallocateCache()
+{
+    // Cache should be empty when we're calling this.
+    assert(cacheCoins.size() == 0);
+    cacheCoins.~CCoinsMap();
+    ::new (&cacheCoins) CCoinsMap();
+}
+
 static const size_t MAX_OUTPUTS_PER_BLOCK = MaxBlockSize() /  ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h.
 
 const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid)

src/coins.h

@@ -331,6 +331,13 @@ class CCoinsViewCache : public CCoinsViewBacked
     //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view
     bool HaveInputs(const CTransaction& tx) const;
 
+    //! Force a reallocation of the cache map. This is required when downsizing
+    //! the cache because the map's allocator may be hanging onto a lot of
+    //! memory despite having called .clear().
+    //!
+    //! See: https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory
+    void ReallocateCache();
+
 private:
     /**
      * @note this is marked const, but may actually append to `cacheCoins`, increasing

src/governance/object.cpp

@@ -533,12 +533,10 @@ bool CGovernanceObject::IsCollateralValid(std::string& strError, bool& fMissingC
     fMissingConfirmations = false;
     uint256 nExpectedHash = GetHash();
 
-    CTransactionRef txCollateral;
-    uint256 nBlockHash;
-
     // RETRIEVE TRANSACTION IN QUESTION
-
-    if (!GetTransaction(nCollateralHash, txCollateral, Params().GetConsensus(), nBlockHash)) {
+    uint256 nBlockHash;
+    CTransactionRef txCollateral = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, nCollateralHash, Params().GetConsensus(), nBlockHash);
+    if (!txCollateral) {
         strError = strprintf("Can't find collateral tx %s", nCollateralHash.ToString());
         LogPrintf("CGovernanceObject::IsCollateralValid -- %s\n", strError);
         return false;

src/init.cpp

@@ -1056,13 +1056,14 @@ static bool InitSanityCheck()
     return true;
 }
 
-static bool AppInitServers(const util::Ref& context)
+static bool AppInitServers(const util::Ref& context, NodeContext& node)
 {
     RPCServer::OnStarted(&OnRPCStarted);
     RPCServer::OnStopped(&OnRPCStopped);
     if (!InitHTTPServer())
         return false;
     StartRPC();
+    node.rpc_interruption_point = RpcInterruptionPoint;
     if (!StartHTTPRPC(context))
         return false;
     if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context);
@@ -1748,7 +1749,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA
     if (gArgs.GetBoolArg("-server", false))
     {
         uiInterface.InitMessage_connect(SetRPCWarmupStatus);
-        if (!AppInitServers(context))
+        if (!AppInitServers(context, node))
             return InitError(_("Unable to start HTTP server. See debug log for details."));
     }
 
@@ -1949,7 +1950,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA
     int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
     nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
     nTotalCache -= nCoinDBCache;
-    nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
+    int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
     int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
     int64_t nEvoDbCache = 1024 * 1024 * 16; // TODO
     LogPrintf("Cache configuration:\n");
@@ -1982,7 +1983,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA
             try {
                 LOCK(cs_main);
                 chainman.InitializeChainstate();
-                UnloadBlockIndex();
+                chainman.m_total_coinstip_cache = nCoinCacheUsage;
+                chainman.m_total_coinsdb_cache = nCoinDBCache;
+
+                UnloadBlockIndex(node.mempool);
 
                 // new CBlockTreeDB tries to delete the existing file, which
                 // fails if it's still open from the previous loop. Close it first:
@@ -1997,7 +2001,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA
                 llmq::quorumSnapshotManager.reset();
                 llmq::quorumSnapshotManager.reset(new llmq::CQuorumSnapshotManager(*evoDb));
 
-                llmq::InitLLMQSystem(*evoDb, *node.connman, false, fReset || fReindexChainState);
+                llmq::InitLLMQSystem(*evoDb, *node.mempool, *node.connman, false, fReset || fReindexChainState);
 
                 if (fReset) {
                     pblocktree->WriteReindexing(true);
@@ -2100,7 +2104,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA
                     }
 
                     // The on-disk coinsdb is now in a good state, create the cache
-                    chainstate->InitCoinsCache();
+                    chainstate->InitCoinsCache(nCoinCacheUsage);
                     assert(chainstate->CanFlushToDisk());
 
                     // flush evodb

src/llmq/chainlocks.cpp

@@ -23,9 +23,9 @@ namespace llmq
 {
 CChainLocksHandler* chainLocksHandler;
 
-CChainLocksHandler::CChainLocksHandler(CConnman& _connman) :
+CChainLocksHandler::CChainLocksHandler(CTxMemPool& _mempool, CConnman& _connman) :
     scheduler(std::make_unique<CScheduler>()),
-    connman(_connman)
+    mempool(_mempool), connman(_connman)
 {
     CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, scheduler.get());
     scheduler_thread = std::make_unique<std::thread>(std::bind(&TraceThread<CScheduler::Function>, "cl-schdlr", serviceLoop));
@@ -647,9 +647,9 @@ void CChainLocksHandler::Cleanup()
         }
     }
     for (auto it = txFirstSeenTime.begin(); it != txFirstSeenTime.end(); ) {
-        CTransactionRef tx;
         uint256 hashBlock;
-        if (!GetTransaction(it->first, tx, Params().GetConsensus(), hashBlock)) {
+        CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, it->first, Params().GetConsensus(), hashBlock);
+        if (!tx) {
             // tx has vanished, probably due to conflicts
             it = txFirstSeenTime.erase(it);
         } else if (!hashBlock.IsNull()) {

src/llmq/chainlocks.h

@@ -22,6 +22,7 @@
 class CConnman;
 class CBlockIndex;
 class CScheduler;
+class CTxMemPool;
 
 namespace llmq
 {
@@ -36,6 +37,7 @@ class CChainLocksHandler : public CRecoveredSigsListener
 
 private:
     CConnman& connman;
+    CTxMemPool& mempool;
     std::unique_ptr<CScheduler> scheduler;
     std::unique_ptr<std::thread> scheduler_thread;
     mutable CCriticalSection cs;
@@ -68,7 +70,7 @@ class CChainLocksHandler : public CRecoveredSigsListener
     int64_t lastCleanupTime GUARDED_BY(cs) {0};
 
 public:
-    explicit CChainLocksHandler(CConnman& _connman);
+    explicit CChainLocksHandler(CTxMemPool& _mempool, CConnman& _connman);
     ~CChainLocksHandler();
 
     void Start();

src/llmq/init.cpp

@@ -23,7 +23,7 @@ namespace llmq
 
 CBLSWorker* blsWorker;
 
-void InitLLMQSystem(CEvoDB& evoDb, CConnman& connman, bool unitTests, bool fWipe)
+void InitLLMQSystem(CEvoDB& evoDb, CTxMemPool& mempool, CConnman& connman, bool unitTests, bool fWipe)
 {
     blsWorker = new CBLSWorker();
 
@@ -33,8 +33,8 @@ void InitLLMQSystem(CEvoDB& evoDb, CConnman& connman, bool unitTests, bool fWipe
     quorumManager = new CQuorumManager(evoDb, connman, *blsWorker, *quorumDKGSessionManager);
     quorumSigSharesManager = new CSigSharesManager(connman);
     quorumSigningManager = new CSigningManager(connman, unitTests, fWipe);
-    chainLocksHandler = new CChainLocksHandler(connman);
-    quorumInstantSendManager = new CInstantSendManager(connman, unitTests, fWipe);
+    chainLocksHandler = new CChainLocksHandler(mempool, connman);
+    quorumInstantSendManager = new CInstantSendManager(mempool, connman, unitTests, fWipe);
 
     // NOTE: we use this only to wipe the old db, do NOT use it for anything else
     // TODO: remove it in some future version

src/llmq/init.h

@@ -8,12 +8,13 @@
 class CConnman;
 class CDBWrapper;
 class CEvoDB;
+class CTxMemPool;
 
 namespace llmq
 {
 
 // Init/destroy LLMQ globals
-void InitLLMQSystem(CEvoDB& evoDb, CConnman& connman, bool unitTests, bool fWipe = false);
+void InitLLMQSystem(CEvoDB& evoDb, CTxMemPool& mempool, CConnman& connman, bool unitTests, bool fWipe = false);
 void DestroyLLMQSystem();
 
 // Manage scheduled tasks, threads, listeners etc.

src/llmq/instantsend.cpp

@@ -64,8 +64,6 @@ void CInstantSendDb::Upgrade()
     if (!db->Read(DB_VERSION, v) || v < CInstantSendDb::CURRENT_VERSION) {
         CDBBatch batch(*db);
         CInstantSendLock islock;
-        CTransactionRef tx;
-        uint256 hashBlock;
 
         auto it = std::unique_ptr<CDBIterator>(db->NewIterator());
         auto firstKey = std::make_tuple(DB_ISLOCK_BY_HASH, uint256());
@@ -76,7 +74,9 @@ void CInstantSendDb::Upgrade()
             if (!it->GetKey(curKey) || std::get<0>(curKey) != DB_ISLOCK_BY_HASH) {
                 break;
             }
-            if (it->GetValue(islock) && !GetTransaction(islock.txid, tx, Params().GetConsensus(), hashBlock)) {
+            uint256 hashBlock;
+            CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, islock.txid, Params().GetConsensus(), hashBlock);
+            if (it->GetValue(islock) && !tx) {
                 // Drop locks for unknown txes
                 batch.Erase(std::make_tuple(DB_HASH_BY_TXID, islock.txid));
                 for (auto& in : islock.inputs) {
@@ -603,10 +603,10 @@ bool CInstantSendManager::CheckCanLock(const COutPoint& outpoint, bool printDebu
         return false;
     }
 
-    CTransactionRef tx;
     uint256 hashBlock;
+    CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, outpoint.hash, params, hashBlock);
     // this relies on enabled txindex and won't work if we ever try to remove the requirement for txindex for masternodes
-    if (!GetTransaction(outpoint.hash, tx, params, hashBlock)) {
+    if (!tx) {
         if (printDebug) {
             LogPrint(BCLog::INSTANTSEND, "CInstantSendManager::%s -- txid=%s: failed to find parent TX %s\n", __func__,
                      txHash.ToString(), outpoint.hash.ToString());
@@ -661,9 +661,9 @@ void CInstantSendManager::HandleNewInputLockRecoveredSig(const CRecoveredSig& re
         g_txindex->BlockUntilSyncedToCurrentChain();
     }
 
-    CTransactionRef tx;
     uint256 hashBlock;
-    if (!GetTransaction(txid, tx, Params().GetConsensus(), hashBlock)) {
+    CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, txid, Params().GetConsensus(), hashBlock);
+    if (!tx) {
         return;
     }
 
@@ -1048,11 +1048,11 @@ void CInstantSendManager::ProcessInstantSendLock(NodeId from, const uint256& has
         return;
     }
 
-    CTransactionRef tx;
     uint256 hashBlock;
+    CTransactionRef tx = GetTransaction(/* block_index */ nullptr, &mempool, islock->txid, Params().GetConsensus(), hashBlock);
     const CBlockIndex* pindexMined{nullptr};
     // we ignore failure here as we must be able to propagate the lock even if we don't have the TX locally
-    if (GetTransaction(islock->txid, tx, Params().GetConsensus(), hashBlock) && !hashBlock.IsNull()) {
+    if (tx && !hashBlock.IsNull()) {
         pindexMined = WITH_LOCK(cs_main, return LookupBlockIndex(hashBlock));
 
         // Let's see if the TX that was locked by this islock is already mined in a ChainLocked block. If yes,

src/llmq/instantsend.h

@@ -13,6 +13,7 @@
 #include <dbwrapper.h>
 #include <primitives/transaction.h>
 #include <threadinterrupt.h>
+#include <txmempool.h>
 
 #include <unordered_map>
 #include <unordered_set>
@@ -194,6 +195,7 @@ class CInstantSendManager : public CRecoveredSigsListener
 private:
     CInstantSendDb db;
     CConnman& connman;
+    CTxMemPool& mempool;
 
     std::atomic<bool> fUpgradedDB{false};
 
@@ -241,7 +243,7 @@ class CInstantSendManager : public CRecoveredSigsListener
     std::unordered_set<uint256, StaticSaltedHasher> pendingRetryTxs GUARDED_BY(cs_pendingRetry);
 
 public:
-    explicit CInstantSendManager(CConnman& _connman, bool unitTests, bool fWipe) : db(unitTests, fWipe), connman(_connman) { workInterrupt.reset(); }
+    explicit CInstantSendManager(CTxMemPool& _mempool, CConnman& _connman, bool unitTests, bool fWipe) : db(unitTests, fWipe), mempool(_mempool), connman(_connman) { workInterrupt.reset(); }
     ~CInstantSendManager() = default;
 
     void Start();

src/net_processing.cpp

@@ -2950,7 +2950,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec
                              pfrom->GetId());
                 } else if (!fAlreadyHave) {
                     bool allowWhileInIBD = allowWhileInIBDObjs.count(inv.type);
-                    if (allowWhileInIBD || (!fImporting && !fReindex && !::ChainstateActive().IsInitialBlockDownload())) {
+                    if (allowWhileInIBD || !chainman.ActiveChainstate().IsInitialBlockDownload()) {
                         RequestObject(State(pfrom->GetId()), inv, current_time);
                     }
                 }

src/node/coinstats.cpp

@@ -85,8 +85,9 @@ static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash,
 
 //! Calculate statistics about the unspent transaction output set
 template <typename T>
-static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj)
+static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point)
 {
+    stats = CCoinsStats();
     std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
     assert(pcursor);
 
@@ -101,7 +102,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj)
     uint256 prevkey;
     std::map<uint32_t, Coin> outputs;
     while (pcursor->Valid()) {
-        boost::this_thread::interruption_point();
+        interruption_point();
         COutPoint key;
         Coin coin;
         if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
@@ -111,6 +112,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj)
             }
             prevkey = key.hash;
             outputs[key.n] = std::move(coin);
+            stats.coins_count++;
         } else {
             return error("%s: unable to read value", __func__);
         }
@@ -126,19 +128,19 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj)
     return true;
 }
 
-bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type)
+bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point)
 {
     switch (hash_type) {
     case(CoinStatsHashType::HASH_SERIALIZED): {
         CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
-        return GetUTXOStats(view, stats, ss);
+        return GetUTXOStats(view, stats, ss, interruption_point);
     }
     case(CoinStatsHashType::MUHASH): {
         MuHash3072 muhash;
-        return GetUTXOStats(view, stats, muhash);
+        return GetUTXOStats(view, stats, muhash, interruption_point);
     }
     case(CoinStatsHashType::NONE): {
-        return GetUTXOStats(view, stats, nullptr);
+        return GetUTXOStats(view, stats, nullptr, interruption_point);
     }
     } // no default case, so the compiler can warn about missing cases
     assert(false);