diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 33941f368100..96dde1668ea2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -22,15 +22,28 @@ int64_t nTimeBestReceived = 0; // Used only to inform the wallet of when we las static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8] +struct IteratorComparator +{ + template + bool operator()(const I& a, const I& b) const + { + return &(*a) < &(*b); + } +}; + struct COrphanTx { + // When modifying, adapt the copy of this definition in tests/DoS_tests. CTransactionRef tx; NodeId fromPeer; + int64_t nTimeExpire; + size_t list_pos; }; +RecursiveMutex g_cs_orphans; +std::map mapOrphanTransactions GUARDED_BY(g_cs_orphans); +std::map::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(g_cs_orphans); +std::vector::iterator> g_orphan_list GUARDED_BY(g_cs_orphans); //! For random eviction -std::map mapOrphanTransactions GUARDED_BY(cs_main); -std::map > mapOrphanTransactionsByPrev GUARDED_BY(cs_main); - -void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +void EraseOrphansFor(NodeId peer); // Internal stuff namespace { @@ -490,7 +503,7 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals) // mapOrphanTransactions // -bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) { const uint256& hash = tx->GetHash(); if (mapOrphanTransactions.count(hash)) @@ -501,66 +514,100 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE // large transaction with a missing parent then we assume // it will rebroadcast it later, after the parent transaction(s) // have been mined or received. - // 10,000 orphans, each of which is at most 5,000 bytes big is - // at most 500 megabytes of orphans: + // 25 orphans, each of which is at most 400,000 bytes big is + // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): unsigned int sz = tx->GetTotalSize(); - if (sz > 5000) { + unsigned int nMaxSize = tx->IsShieldedTx() ? MAX_TX_SIZE_AFTER_SAPLING : MAX_STANDARD_TX_SIZE; + if (sz >= nMaxSize) { LogPrint(BCLog::MEMPOOL, "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; } - auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer}); + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME, g_orphan_list.size()}); assert(ret.second); - for (const CTxIn& txin : tx->vin) - mapOrphanTransactionsByPrev[txin.prevout.hash].insert(hash); + g_orphan_list.emplace_back(ret.first); + for (const CTxIn& txin : tx->vin) { + mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); + } - LogPrint(BCLog::MEMPOOL, "stored orphan tx %s (mapsz %u prevsz %u)\n", hash.ToString(), + LogPrint(BCLog::MEMPOOL, "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } -void static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) { std::map::iterator it = mapOrphanTransactions.find(hash); if (it == mapOrphanTransactions.end()) - return; + return 0; for (const CTxIn& txin : it->second.tx->vin) { - std::map >::iterator itPrev = mapOrphanTransactionsByPrev.find(txin.prevout.hash); + auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itPrev == mapOrphanTransactionsByPrev.end()) continue; - itPrev->second.erase(hash); + itPrev->second.erase(it); if (itPrev->second.empty()) mapOrphanTransactionsByPrev.erase(itPrev); } + + size_t old_pos = it->second.list_pos; + assert(g_orphan_list[old_pos] == it); + if (old_pos + 1 != g_orphan_list.size()) { + // Unless we're deleting the last entry in g_orphan_list, move the last + // entry to the position we're deleting. + auto it_last = g_orphan_list.back(); + g_orphan_list[old_pos] = it_last; + it_last->second.list_pos = old_pos; + } + g_orphan_list.pop_back(); + mapOrphanTransactions.erase(it); + return 1; } -void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void EraseOrphansFor(NodeId peer) { + LOCK(g_cs_orphans); int nErased = 0; std::map::iterator iter = mapOrphanTransactions.begin(); while (iter != mapOrphanTransactions.end()) { std::map::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { - EraseOrphanTx(maybeErase->second.tx->GetHash()); - ++nErased; + nErased += EraseOrphanTx(maybeErase->second.tx->GetHash()); } } if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer %d\n", nErased, peer); } -unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) { + LOCK(g_cs_orphans); + unsigned int nEvicted = 0; + static int64_t nNextSweep; + int64_t nNow = GetTime(); + if (nNextSweep <= nNow) { + // Sweep out expired orphan pool entries: + int nErased = 0; + int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; + auto iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) { + auto maybeErase = iter++; + if (maybeErase->second.nTimeExpire <= nNow) { + nErased += EraseOrphanTx(maybeErase->second.tx->GetHash()); + } else { + nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); + } + } + // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. + nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx due to expiration\n", nErased); + } FastRandomContext rng; while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = rng.rand256(); - std::map::iterator it = mapOrphanTransactions.lower_bound(randomhash); - if (it == mapOrphanTransactions.end()) - it = mapOrphanTransactions.begin(); - EraseOrphanTx(it->first); + size_t randompos = rng.randrange(g_orphan_list.size()); + EraseOrphanTx(g_orphan_list[randompos]->first); ++nEvicted; } return nEvicted; @@ -601,6 +648,37 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); } +void PeerLogicValidation::BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindex) +{ + LOCK(g_cs_orphans); + + std::vector vOrphanErase; + + for (const CTransactionRef& ptx : pblock->vtx) { + const CTransaction& tx = *ptx; + + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = *(*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.emplace_back(orphanHash); + } + } + } + + // Erase orphan transactions include or precluded by this block + if (!vOrphanErase.empty()) { + int nErased = 0; + for (uint256& orphanHash : vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); + } +} + void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex* pindexNew, const CBlockIndex* pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; @@ -661,10 +739,13 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } + { + LOCK(g_cs_orphans); + if (mapOrphanTransactions.count(inv.hash)) return true; + } return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || - mapOrphanTransactions.count(inv.hash) || pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } @@ -1413,7 +1494,7 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR else if (strCommand == NetMsgType::TX) { - std::vector vWorkQueue; + std::deque vWorkQueue; std::vector vEraseQueue; CTransaction tx(deserialize, vRecv); CTransactionRef ptx = MakeTransactionRef(tx); @@ -1421,19 +1502,26 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR CInv inv(MSG_TX, tx.GetHash()); pfrom->AddInventoryKnown(inv); - LOCK(cs_main); + LOCK2(cs_main, g_cs_orphans); bool ignoreFees = false; bool fMissingInputs = false; - bool fMissingZerocoinInputs = false; CValidationState state; mapAlreadyAskedFor.erase(inv); - if (!tx.HasZerocoinSpendInputs() && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs, false, ignoreFees)) { + if (ptx->ContainsZerocoins()) { + // Don't even try to check zerocoins at all. + Misbehaving(pfrom->GetId(), 100); + LogPrint(BCLog::NET, " misbehaving peer, received a zc transaction, peer: %s\n", pfrom->GetAddrName()); + } + + if (AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs, false, ignoreFees)) { mempool.check(pcoinsTip); RelayTransaction(tx, connman); - vWorkQueue.push_back(inv.hash); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + vWorkQueue.emplace_back(inv.hash, i); + } LogPrint(BCLog::MEMPOOL, "%s : peer=%d %s : accepted %s (poolsz %u txn, %u kB)\n", __func__, pfrom->id, pfrom->cleanSubVer, tx.GetHash().ToString(), @@ -1441,16 +1529,17 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR // Recursively process any orphan transactions that depended on this one std::set setMisbehaving; - for(unsigned int i = 0; i < vWorkQueue.size(); i++) { - std::map >::iterator itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue[i]); + while (!vWorkQueue.empty()) { + auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); + vWorkQueue.pop_front(); if(itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for(std::set::iterator mi = itByPrev->second.begin(); + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const uint256 &orphanHash = *mi; - const auto &orphanTx = mapOrphanTransactions[orphanHash].tx; - NodeId fromPeer = mapOrphanTransactions[orphanHash].fromPeer; + const CTransactionRef& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx->GetHash(); + NodeId fromPeer = (*mi)->second.fromPeer; bool fMissingInputs2 = false; // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get @@ -1458,14 +1547,16 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR CValidationState stateDummy; - if(setMisbehaving.count(fromPeer)) + if (setMisbehaving.count(fromPeer)) continue; - if(AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { + if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(*orphanTx, connman); - vWorkQueue.push_back(orphanHash); + for (unsigned int i = 0; i < orphanTx->vout.size(); i++) { + vWorkQueue.emplace_back(orphanHash, i); + } vEraseQueue.push_back(orphanHash); - } else if(!fMissingInputs2) { + } else if (!fMissingInputs2) { int nDos = 0; if(stateDummy.IsInvalid(nDos) && nDos > 0) { // Punish peer that gave us an invalid orphan tx @@ -1486,22 +1577,41 @@ bool static ProcessMessage(CNode* pfrom, std::string strCommand, CDataStream& vR for (uint256& hash : vEraseQueue) EraseOrphanTx(hash); - } else if (tx.HasZerocoinSpendInputs() && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingZerocoinInputs, false, false, ignoreFees)) { - //Presstab: ZCoin has a bunch of code commented out here. Is this something that should have more going on? - //Also there is nothing that handles fMissingZerocoinInputs. Does there need to be? - RelayTransaction(tx, connman); - LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: Zerocoinspend peer=%d %s : accepted %s (poolsz %u)\n", - pfrom->id, pfrom->cleanSubVer, - tx.GetHash().ToString(), - mempool.mapTx.size()); } else if (fMissingInputs) { - AddOrphanTx(ptx, pfrom->GetId()); + bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + + // Deduplicate parent txids, so that we don't have to loop over + // the same parent txid more than once down below. + std::vector unique_parents; + unique_parents.reserve(tx.vin.size()); + for (const CTxIn& txin : ptx->vin) { + // We start with all parents, and then remove duplicates below. + unique_parents.emplace_back(txin.prevout.hash); + } + std::sort(unique_parents.begin(), unique_parents.end()); + unique_parents.erase(std::unique(unique_parents.begin(), unique_parents.end()), unique_parents.end()); + for (const uint256& parent_txid : unique_parents) { + if (recentRejects->contains(parent_txid)) { + fRejectedParents = true; + break; + } + } + if (!fRejectedParents) { + for (const uint256& parent_txid : unique_parents) { + CInv _inv(MSG_TX, parent_txid); + pfrom->AddInventoryKnown(_inv); + if (!AlreadyHave(_inv)) pfrom->AskFor(_inv); + } + AddOrphanTx(ptx, pfrom->GetId()); - // DoS prevention: do not allow mapOrphanTransactions to grow unbounded - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint(BCLog::MEMPOOL, "mapOrphan overflow, removed %u tx\n", nEvicted); + // DoS prevention: do not allow mapOrphanTransactions to grow unbounded + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); + if (nEvicted > 0) + LogPrint(BCLog::MEMPOOL, "mapOrphan overflow, removed %u tx\n", nEvicted); + } else { + LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + } } else { // AcceptToMemoryPool() returned false, possibly because the tx is // already in the mempool; if the tx isn't in the mempool that diff --git a/src/net_processing.h b/src/net_processing.h index bc7abeb5ebdf..3b20878b0c0f 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -13,7 +13,11 @@ extern RecursiveMutex cs_main; // !TODO: change mutex to cs_orphans /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ -static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; +static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 25; +/** Expiration time for orphan transactions in seconds */ +static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; +/** Minimum time between orphan transactions expire time checks in seconds */ +static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; /** Default for -blockspamfilter, use header spam filter */ static const bool DEFAULT_BLOCK_SPAM_FILTER = true; /** Default for -blockspamfiltermaxsize, maximum size of the list of indexes in the block spam filter */ @@ -41,6 +45,7 @@ class PeerLogicValidation : public CValidationInterface { PeerLogicValidation(CConnman* connmanIn); ~PeerLogicValidation() = default; + void BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindex) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; void BlockChecked(const CBlock& block, const CValidationState& state) override; }; diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index e3d0f01076dd..349af79763f2 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -31,9 +31,10 @@ extern unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans); struct COrphanTx { CTransactionRef tx; NodeId fromPeer; + int64_t nTimeExpire; }; -extern std::map mapOrphanTransactions; -extern std::map > mapOrphanTransactionsByPrev; +extern RecursiveMutex g_cs_orphans; +extern std::map mapOrphanTransactions GUARDED_BY(g_cs_orphans); CService ip(uint32_t i) { @@ -135,6 +136,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) CTransactionRef RandomOrphan() { std::map::iterator it; + LOCK2(cs_main, g_cs_orphans); it = mapOrphanTransactions.lower_bound(InsecureRand256()); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); @@ -205,7 +207,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) tx.vout.resize(1); tx.vout[0].nValue = 1*CENT; tx.vout[0].scriptPubKey = GetScriptForDestination(key.GetPubKey().GetID()); - tx.vin.resize(500); + tx.vin.resize(2777); for (unsigned int j = 0; j < tx.vin.size(); j++) { tx.vin[j].prevout.n = j; @@ -220,6 +222,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(!AddOrphanTx(MakeTransactionRef(tx), i)); } + LOCK2(cs_main, g_cs_orphans); // Test EraseOrphansFor: for (NodeId i = 0; i < 3; i++) { @@ -235,7 +238,6 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(mapOrphanTransactions.size() <= 10); LimitOrphanTxSize(0); BOOST_CHECK(mapOrphanTransactions.empty()); - BOOST_CHECK(mapOrphanTransactionsByPrev.empty()); } BOOST_AUTO_TEST_SUITE_END()