diff --git a/src/chain.h b/src/chain.h index 079835e495d9..5438d8f3d304 100644 --- a/src/chain.h +++ b/src/chain.h @@ -14,6 +14,60 @@ #include +class CBlockFileInfo +{ +public: + unsigned int nBlocks; //!< number of blocks stored in file + unsigned int nSize; //!< number of used bytes of block file + unsigned int nUndoSize; //!< number of used bytes in the undo file + unsigned int nHeightFirst; //!< lowest height of block in file + unsigned int nHeightLast; //!< highest height of block in file + uint64_t nTimeFirst; //!< earliest time of block in file + uint64_t nTimeLast; //!< latest time of block in file + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(VARINT(nBlocks)); + READWRITE(VARINT(nSize)); + READWRITE(VARINT(nUndoSize)); + READWRITE(VARINT(nHeightFirst)); + READWRITE(VARINT(nHeightLast)); + READWRITE(VARINT(nTimeFirst)); + READWRITE(VARINT(nTimeLast)); + } + + void SetNull() { + nBlocks = 0; + nSize = 0; + nUndoSize = 0; + nHeightFirst = 0; + nHeightLast = 0; + nTimeFirst = 0; + nTimeLast = 0; + } + + CBlockFileInfo() { + SetNull(); + } + + std::string ToString() const; + + /** update statistics (does not update nSize) */ + void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) { + if (nBlocks==0 || nHeightFirst > nHeightIn) + nHeightFirst = nHeightIn; + if (nBlocks==0 || nTimeFirst > nTimeIn) + nTimeFirst = nTimeIn; + nBlocks++; + if (nHeightIn > nHeightLast) + nHeightLast = nHeightIn; + if (nTimeIn > nTimeLast) + nTimeLast = nTimeIn; + } +}; + struct CDiskBlockPos { int nFile; diff --git a/src/coins.cpp b/src/coins.cpp index 4d1dbdea4ed3..39db7dedfbbe 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -45,7 +45,7 @@ bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return fal bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } -bool CCoinsView::GetStats(CCoinsStats &stats) const { return false; } +CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } @@ -54,9 +54,9 @@ bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveC uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } -bool CCoinsViewBacked::GetStats(CCoinsStats &stats) const { return base->GetStats(stats); } +CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } -CCoinsKeyHasher::CCoinsKeyHasher() : salt(GetRandHash()) {} +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } @@ -117,11 +117,17 @@ CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { return CCoinsModifier(*this, ret.first, cachedCoinUsage); } -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid) { +// ModifyNewCoins has to know whether the new outputs its creating are for a +// coinbase or not. If they are for a coinbase, it can not mark them as fresh. +// This is to ensure that the historical duplicate coinbases before BIP30 was +// in effect will still be properly overwritten when spent. +CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { assert(!hasModifier); std::pair ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; + if (!coinbase) { + ret.first->second.flags = CCoinsCacheEntry::FRESH; + } ret.first->second.flags |= CCoinsCacheEntry::DIRTY; return CCoinsModifier(*this, ret.first, 0); } @@ -294,3 +300,7 @@ CCoinsModifier::~CCoinsModifier() cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); } } + +CCoinsViewCursor::~CCoinsViewCursor() +{ +} diff --git a/src/coins.h b/src/coins.h index 4885bfe0a847..5810790818e3 100644 --- a/src/coins.h +++ b/src/coins.h @@ -8,6 +8,7 @@ #include "compressor.h" #include "core_memusage.h" +#include "hash.h" #include "memusage.h" #include "serialize.h" #include "uint256.h" @@ -264,21 +265,22 @@ class CCoins } }; -class CCoinsKeyHasher +class SaltedTxidHasher { private: - uint256 salt; + /** Salt */ + const uint64_t k0, k1; public: - CCoinsKeyHasher(); + SaltedTxidHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& key) const { - return key.GetHash(salt); + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); } }; @@ -295,21 +297,28 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : coins(), flags(0) {} }; -typedef boost::unordered_map CCoinsMap; +typedef boost::unordered_map CCoinsMap; -struct CCoinsStats +/** Cursor for iterating over CoinsView state */ +class CCoinsViewCursor { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nSerializedSize; - uint256 hashSerialized; - CAmount nTotalAmount; +public: + CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} + virtual ~CCoinsViewCursor(); - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} -}; + virtual bool GetKey(uint256 &key) const = 0; + virtual bool GetValue(CCoins &coins) const = 0; + /* Don't care about GetKeySize here */ + virtual unsigned int GetValueSize() const = 0; + virtual bool Valid() const = 0; + virtual void Next() = 0; + + //! Get best block at the time this cursor was created + const uint256 &GetBestBlock() const { return hashBlock; } +private: + uint256 hashBlock; +}; /** Abstract view on the open txout dataset. */ class CCoinsView @@ -329,8 +338,8 @@ class CCoinsView //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - //! Calculate statistics about the unspent transaction output set - virtual bool GetStats(CCoinsStats &stats) const; + //! Get a cursor to iterate over the whole state + virtual CCoinsViewCursor *Cursor() const; //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} @@ -350,7 +359,7 @@ class CCoinsViewBacked : public CCoinsView uint256 GetBestBlock() const; void SetBackend(CCoinsView &viewIn); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - bool GetStats(CCoinsStats &stats) const; + CCoinsViewCursor *Cursor() const; }; @@ -435,7 +444,7 @@ class CCoinsViewCache : public CCoinsViewBacked * would not properly overwrite the first coinbase of the pair. Simultaneous modifications * are not allowed. */ - CCoinsModifier ModifyNewCoins(const uint256 &txid); + CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); /** * Push the modifications applied to this cache to its base. diff --git a/src/compressor.h b/src/compressor.h index 4a7209083058..fa702f0dfa90 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -86,8 +86,14 @@ class CScriptCompressor return; } nSize -= nSpecialScripts; - script.resize(nSize); - s >> REF(CFlatData(script)); + if (nSize > MAX_SCRIPT_SIZE) { + // Overly long script, replace with a short invalid one + script << OP_RETURN; + s.ignore(nSize); + } else { + script.resize(nSize); + s >> REF(CFlatData(script)); + } } }; diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 1907e2fa7843..09c68fbe550c 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -15,20 +15,6 @@ #include #include -void HandleError(const leveldb::Status& status) throw(dbwrapper_error) -{ - if (status.ok()) - return; - LogPrintf("%s\n", status.ToString()); - if (status.IsCorruption()) - throw dbwrapper_error("Database corrupted"); - if (status.IsIOError()) - throw dbwrapper_error("Database I/O error"); - if (status.IsNotFound()) - throw dbwrapper_error("Database entry missing"); - throw dbwrapper_error("Unknown database error"); -} - static leveldb::Options GetOptions(size_t nCacheSize) { leveldb::Options options; @@ -61,13 +47,13 @@ CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, b if (fWipe) { LogPrintf("Wiping LevelDB in %s\n", path.string()); leveldb::Status result = leveldb::DestroyDB(path.string(), options); - HandleError(result); + dbwrapper_private::HandleError(result); } TryCreateDirectory(path); LogPrintf("Opening LevelDB in %s\n", path.string()); } leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); - HandleError(status); + dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); // The base-case obfuscation key, which is a noop. @@ -84,10 +70,10 @@ CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, b Write(OBFUSCATE_KEY_KEY, new_key); obfuscate_key = new_key; - LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key)); } - LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex()); + LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key)); } CDBWrapper::~CDBWrapper() @@ -102,10 +88,10 @@ CDBWrapper::~CDBWrapper() options.env = NULL; } -bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) throw(dbwrapper_error) +bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) { leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); - HandleError(status); + dbwrapper_private::HandleError(status); return true; } @@ -136,17 +122,30 @@ bool CDBWrapper::IsEmpty() return !(it->Valid()); } -const std::vector& CDBWrapper::GetObfuscateKey() const +CDBIterator::~CDBIterator() { delete piter; } +bool CDBIterator::Valid() { return piter->Valid(); } +void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CDBIterator::Next() { piter->Next(); } + +namespace dbwrapper_private { + +void HandleError(const leveldb::Status& status) { - return obfuscate_key; + if (status.ok()) + return; + LogPrintf("%s\n", status.ToString()); + if (status.IsCorruption()) + throw dbwrapper_error("Database corrupted"); + if (status.IsIOError()) + throw dbwrapper_error("Database I/O error"); + if (status.IsNotFound()) + throw dbwrapper_error("Database entry missing"); + throw dbwrapper_error("Unknown database error"); } -std::string CDBWrapper::GetObfuscateKeyHex() const +const std::vector& GetObfuscateKey(const CDBWrapper &w) { - return HexStr(obfuscate_key); + return w.obfuscate_key; } -CDBIterator::~CDBIterator() { delete piter; } -bool CDBIterator::Valid() { return piter->Valid(); } -void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CDBIterator::Next() { piter->Next(); } +}; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5e7313f7eb54..a0779d3ab991 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -23,7 +23,23 @@ class dbwrapper_error : public std::runtime_error dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} }; -void HandleError(const leveldb::Status& status) throw(dbwrapper_error); +class CDBWrapper; + +/** These should be considered an implementation detail of the specific database. + */ +namespace dbwrapper_private { + +/** Handle database error by throwing dbwrapper_error exception. + */ +void HandleError(const leveldb::Status& status); + +/** Work around circular dependency, as well as for testing in dbwrapper_tests. + * Database obfuscation should be considered an implementation detail of the + * specific database. + */ +const std::vector& GetObfuscateKey(const CDBWrapper &w); + +}; /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch @@ -31,14 +47,14 @@ class CDBBatch friend class CDBWrapper; private: + const CDBWrapper &parent; leveldb::WriteBatch batch; - const std::vector *obfuscate_key; public: /** - * @param[in] obfuscate_key If passed, XOR data with this key. + * @param[in] parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const std::vector *obfuscate_key) : obfuscate_key(obfuscate_key) { }; + CDBBatch(const CDBWrapper &parent) : parent(parent) { }; template void Write(const K& key, const V& value) @@ -51,7 +67,7 @@ class CDBBatch CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(ssValue.GetSerializeSize(value)); ssValue << value; - ssValue.Xor(*obfuscate_key); + ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(&ssValue[0], ssValue.size()); batch.Put(slKey, slValue); @@ -72,17 +88,17 @@ class CDBBatch class CDBIterator { private: + const CDBWrapper &parent; leveldb::Iterator *piter; - const std::vector *obfuscate_key; public: /** + * @param[in] parent Parent CDBWrapper instance. * @param[in] piterIn The original leveldb iterator. - * @param[in] obfuscate_key If passed, XOR data with this key. */ - CDBIterator(leveldb::Iterator *piterIn, const std::vector* obfuscate_key) : - piter(piterIn), obfuscate_key(obfuscate_key) { }; + CDBIterator(const CDBWrapper &parent, leveldb::Iterator *piterIn) : + parent(parent), piter(piterIn) { }; ~CDBIterator(); bool Valid(); @@ -118,7 +134,7 @@ class CDBIterator leveldb::Slice slValue = piter->value(); try { CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); - ssValue.Xor(*obfuscate_key); + ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { return false; @@ -134,6 +150,7 @@ class CDBIterator class CDBWrapper { + friend const std::vector& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w); private: //! custom environment this database is using (may be NULL in case of default environment) leveldb::Env* penv; @@ -180,7 +197,7 @@ class CDBWrapper ~CDBWrapper(); template - bool Read(const K& key, V& value) const throw(dbwrapper_error) + bool Read(const K& key, V& value) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(ssKey.GetSerializeSize(key)); @@ -193,7 +210,7 @@ class CDBWrapper if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); @@ -206,15 +223,15 @@ class CDBWrapper } template - bool Write(const K& key, const V& value, bool fSync = false) throw(dbwrapper_error) + bool Write(const K& key, const V& value, bool fSync = false) { - CDBBatch batch(&obfuscate_key); + CDBBatch batch(*this); batch.Write(key, value); return WriteBatch(batch, fSync); } template - bool Exists(const K& key) const throw(dbwrapper_error) + bool Exists(const K& key) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(ssKey.GetSerializeSize(key)); @@ -227,20 +244,20 @@ class CDBWrapper if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } return true; } template - bool Erase(const K& key, bool fSync = false) throw(dbwrapper_error) + bool Erase(const K& key, bool fSync = false) { - CDBBatch batch(&obfuscate_key); + CDBBatch batch(*this); batch.Erase(key); return WriteBatch(batch, fSync); } - bool WriteBatch(CDBBatch& batch, bool fSync = false) throw(dbwrapper_error); + bool WriteBatch(CDBBatch& batch, bool fSync = false); // not available for LevelDB; provide for compatibility with BDB bool Flush() @@ -248,32 +265,21 @@ class CDBWrapper return true; } - bool Sync() throw(dbwrapper_error) + bool Sync() { - CDBBatch batch(&obfuscate_key); + CDBBatch batch(*this); return WriteBatch(batch, true); } CDBIterator *NewIterator() { - return new CDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key); + return new CDBIterator(*this, pdb->NewIterator(iteroptions)); } /** * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); - - /** - * Accessor for obfuscate_key. - */ - const std::vector& GetObfuscateKey() const; - - /** - * Return the obfuscate_key as a hex-formatted string. - */ - std::string GetObfuscateKeyHex() const; - }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/hash.cpp b/src/hash.cpp index 7f3cf1a1faa7..a518314a5376 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -81,3 +81,97 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he num[3] = (nChild >> 0) & 0xFF; CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } + +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) + +#define SIPROUND do { \ + v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \ + v2 = ROTL(v2, 32); \ +} while (0) + +CSipHasher::CSipHasher(uint64_t k0, uint64_t k1) +{ + v[0] = 0x736f6d6570736575ULL ^ k0; + v[1] = 0x646f72616e646f6dULL ^ k1; + v[2] = 0x6c7967656e657261ULL ^ k0; + v[3] = 0x7465646279746573ULL ^ k1; + count = 0; +} + +CSipHasher& CSipHasher::Write(uint64_t data) +{ + uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + + v3 ^= data; + SIPROUND; + SIPROUND; + v0 ^= data; + + v[0] = v0; + v[1] = v1; + v[2] = v2; + v[3] = v3; + + count++; + return *this; +} + +uint64_t CSipHasher::Finalize() const +{ + uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + + v3 ^= ((uint64_t)count) << 59; + SIPROUND; + SIPROUND; + v0 ^= ((uint64_t)count) << 59; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} + +uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v3 ^= ((uint64_t)4) << 59; + SIPROUND; + SIPROUND; + v0 ^= ((uint64_t)4) << 59; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index dfc44a133a9b..14e40797db77 100644 --- a/src/hash.h +++ b/src/hash.h @@ -278,6 +278,21 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector inline uint256 HashX11(const T1 pbegin, const T1 pend) diff --git a/src/init.cpp b/src/init.cpp index 19d1f9b1dfa8..acb203c3ef92 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1511,10 +1511,10 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache int64_t nBlockTreeDBCache = nTotalCache / 8; - if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) - nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB + nBlockTreeDBCache = std::min(nBlockTreeDBCache, (GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxBlockDBAndTxIndexCache : nMaxBlockDBCache) << 20); nTotalCache -= nBlockTreeDBCache; int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache + nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache LogPrintf("Cache configuration:\n"); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d986759758a8..ebfd673b4e88 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -691,10 +691,12 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } + // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude + // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoins(inv.hash); + pcoinsTip->HaveCoinsInCache(inv.hash); } case MSG_BLOCK: diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 281920ee1e2d..76898ae2827c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -19,11 +19,14 @@ #include "txmempool.h" #include "util.h" #include "utilstrencodings.h" +#include "hash.h" #include #include +#include // boost::thread::interrupt + using namespace std; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); @@ -550,6 +553,59 @@ UniValue getblock(const UniValue& params, bool fHelp) return blockToJSON(block, pblockindex); } +struct CCoinsStats +{ + int nHeight; + uint256 hashBlock; + uint64_t nTransactions; + uint64_t nTransactionOutputs; + uint64_t nSerializedSize; + uint256 hashSerialized; + CAmount nTotalAmount; + + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} +}; + +//! Calculate statistics about the unspent transaction output set +static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) +{ + boost::scoped_ptr pcursor(view->Cursor()); + + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + stats.hashBlock = pcursor->GetBestBlock(); + { + LOCK(cs_main); + stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; + } + ss << stats.hashBlock; + CAmount nTotalAmount = 0; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + uint256 key; + CCoins coins; + if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { + stats.nTransactions++; + for (unsigned int i=0; iGetValueSize(); + ss << VARINT(0); + } else { + return error("%s: unable to read value", __func__); + } + pcursor->Next(); + } + stats.hashSerialized = ss.GetHash(); + stats.nTotalAmount = nTotalAmount; + return true; +} + UniValue gettxoutsetinfo(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 0) @@ -576,7 +632,7 @@ UniValue gettxoutsetinfo(const UniValue& params, bool fHelp) CCoinsStats stats; FlushStateToDisk(); - if (pcoinsTip->GetStats(stats)) { + if (GetUTXOStats(pcoinsTip, stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 149a4f015631..9f29ee7815a2 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -247,7 +247,7 @@ bool EvalScript(vector >& stack, const CScript& script, un vector vfExec; vector altstack; set_error(serror, SCRIPT_ERR_UNKNOWN_ERROR); - if (script.size() > 10000) + if (script.size() > MAX_SCRIPT_SIZE) return set_error(serror, SCRIPT_ERR_SCRIPT_SIZE); int nOpCount = 0; bool fRequireMinimal = (flags & SCRIPT_VERIFY_MINIMALDATA) != 0; diff --git a/src/script/script.h b/src/script/script.h index 32cb200a9f47..f1b32cbb5460 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -27,6 +27,9 @@ static const int MAX_OPS_PER_SCRIPT = 201; // Maximum number of public keys per multisig static const int MAX_PUBKEYS_PER_MULTISIG = 20; +// Maximum script length in bytes +static const int MAX_SCRIPT_SIZE = 10000; + // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. static const unsigned int LOCKTIME_THRESHOLD = 500000000; // Tue Nov 5 00:53:20 1985 UTC @@ -633,7 +636,7 @@ class CScript : public CScriptBase */ bool IsUnspendable() const { - return (size() > 0 && *begin() == OP_RETURN); + return (size() > 0 && *begin() == OP_RETURN) || (size() > MAX_SCRIPT_SIZE); } void clear() diff --git a/src/spentindex.h b/src/spentindex.h index bd5da45d60db..eb5fcce9120a 100644 --- a/src/spentindex.h +++ b/src/spentindex.h @@ -8,6 +8,7 @@ #include "uint256.h" #include "amount.h" +#include "script/script.h" struct CSpentIndexKey { uint256 txid; @@ -95,4 +96,278 @@ struct CSpentIndexKeyCompare } }; +struct CTimestampIndexIteratorKey { + unsigned int timestamp; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + } + + CTimestampIndexIteratorKey(unsigned int time) { + timestamp = time; + } + + CTimestampIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + } +}; + +struct CTimestampIndexKey { + unsigned int timestamp; + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 36; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + blockHash.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampIndexKey(unsigned int time, uint256 hash) { + timestamp = time; + blockHash = hash; + } + + CTimestampIndexKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + blockHash.SetNull(); + } +}; + +struct CAddressUnspentKey { + unsigned int type; + uint160 hashBytes; + uint256 txhash; + size_t index; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 57; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, index); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + txhash.Unserialize(s, nType, nVersion); + index = ser_readdata32(s); + } + + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, uint256 txid, size_t indexValue) { + type = addressType; + hashBytes = addressHash; + txhash = txid; + index = indexValue; + } + + CAddressUnspentKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + txhash.SetNull(); + index = 0; + } +}; + +struct CAddressUnspentValue { + CAmount satoshis; + CScript script; + int blockHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(satoshis); + READWRITE(*(CScriptBase*)(&script)); + READWRITE(blockHeight); + } + + CAddressUnspentValue(CAmount sats, CScript scriptPubKey, int height) { + satoshis = sats; + script = scriptPubKey; + blockHeight = height; + } + + CAddressUnspentValue() { + SetNull(); + } + + void SetNull() { + satoshis = -1; + script.clear(); + blockHeight = 0; + } + + bool IsNull() const { + return (satoshis == -1); + } +}; + +struct CAddressIndexKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; + uint256 txhash; + size_t index; + bool spending; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 66; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, index); + char f = spending; + ser_writedata8(s, f); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + index = ser_readdata32(s); + char f = ser_readdata8(s); + spending = f; + } + + CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t indexValue, bool isSpending) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; + txhash = txid; + index = indexValue; + spending = isSpending; + } + + CAddressIndexKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; + txhash.SetNull(); + index = 0; + spending = false; + } + +}; + +struct CAddressIndexIteratorKey { + unsigned int type; + uint160 hashBytes; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 21; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { + type = addressType; + hashBytes = addressHash; + } + + CAddressIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + } +}; + +struct CAddressIndexIteratorHeightKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 25; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + ser_writedata32be(s, blockHeight); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + } + + CAddressIndexIteratorHeightKey(unsigned int addressType, uint160 addressHash, int height) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + } + + CAddressIndexIteratorHeightKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; + } +}; + + #endif // BITCOIN_SPENTINDEX_H diff --git a/src/streams.h b/src/streams.h index 3d887feb3b76..664813910593 100644 --- a/src/streams.h +++ b/src/streams.h @@ -247,7 +247,9 @@ class CDataStream CDataStream& ignore(int nSize) { // Ignore from the beginning of the buffer - assert(nSize >= 0); + if (nSize < 0) { + throw std::ios_base::failure("CDataStream::ignore(): nSize negative"); + } unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) { @@ -411,6 +413,20 @@ class CAutoFile return (*this); } + CAutoFile& ignore(size_t nSize) + { + if (!file) + throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL"); + unsigned char data[4096]; + while (nSize > 0) { + size_t nNow = std::min(nSize, sizeof(data)); + if (fread(data, 1, nNow, file) != nNow) + throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); + nSize -= nNow; + } + return (*this); + } + CAutoFile& write(const char* pch, size_t nSize) { if (!file) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index fb68f22c5cff..e54429e0d6a5 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -4,7 +4,9 @@ #include "coins.h" #include "random.h" +#include "script/standard.h" #include "uint256.h" +#include "utilstrencodings.h" #include "test/test_dash.h" #include "validation.h" #include "consensus/validation.h" @@ -61,8 +63,6 @@ class CCoinsViewTest : public CCoinsView hashBestBlock_ = hashBlock; return true; } - - bool GetStats(CCoinsStats& stats) const { return false; } }; class CCoinsViewCacheTest : public CCoinsViewCache @@ -347,4 +347,73 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_CHECK(spent_a_duplicate_coinbase); } +BOOST_AUTO_TEST_CASE(ccoins_serialization) +{ + // Good example + CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); + CCoins cc1; + ss1 >> cc1; + BOOST_CHECK_EQUAL(cc1.nVersion, 1); + BOOST_CHECK_EQUAL(cc1.fCoinBase, false); + BOOST_CHECK_EQUAL(cc1.nHeight, 203998); + BOOST_CHECK_EQUAL(cc1.vout.size(), 2); + BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); + BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); + BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + + // Good example + CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); + CCoins cc2; + ss2 >> cc2; + BOOST_CHECK_EQUAL(cc2.nVersion, 1); + BOOST_CHECK_EQUAL(cc2.fCoinBase, true); + BOOST_CHECK_EQUAL(cc2.nHeight, 120891); + BOOST_CHECK_EQUAL(cc2.vout.size(), 17); + for (int i = 0; i < 17; i++) { + BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); + } + BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); + BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); + BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + + // Smallest possible example + CDataStream ssx(SER_DISK, CLIENT_VERSION); + BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); + + CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); + CCoins cc3; + ss3 >> cc3; + BOOST_CHECK_EQUAL(cc3.nVersion, 0); + BOOST_CHECK_EQUAL(cc3.fCoinBase, false); + BOOST_CHECK_EQUAL(cc3.nHeight, 0); + BOOST_CHECK_EQUAL(cc3.vout.size(), 1); + BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); + BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); + BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + + // scriptPubKey that ends beyond the end of the stream + CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + try { + CCoins cc4; + ss4 >> cc4; + BOOST_CHECK_MESSAGE(false, "We should have thrown"); + } catch (const std::ios_base::failure& e) { + } + + // Very large scriptPubKey (3*10^9 bytes) past the end of the stream + CDataStream tmp(SER_DISK, CLIENT_VERSION); + uint64_t x = 3000000000ULL; + tmp << VARINT(x); + BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); + CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + try { + CCoins cc5; + ss5 >> cc5; + BOOST_CHECK_MESSAGE(false, "We should have thrown"); + } catch (const std::ios_base::failure& e) { + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 4b089d32e36b..e02654ed2e2a 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper) uint256 res; // Ensure that we're doing real obfuscation when obfuscate=true - BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey())); + BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw))); BOOST_CHECK(dbw.Write(key, in)); BOOST_CHECK(dbw.Read(key, res)); @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) uint256 in3 = GetRandHash(); uint256 res; - CDBBatch batch(&dbw.GetObfuscateKey()); + CDBBatch batch(dbw); batch.Write(key, in); batch.Write(key2, in2); @@ -156,7 +156,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data - BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string + BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string uint256 in2 = GetRandHash(); uint256 res3; @@ -193,7 +193,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) // Check that the key/val we wrote with unobfuscated wrapper doesn't exist uint256 res2; BOOST_CHECK(!odbw.Read(key, res2)); - BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey())); + BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); uint256 in2 = GetRandHash(); uint256 res3; diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 13939a7090f7..073b2f6716a3 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -47,4 +47,24 @@ BOOST_AUTO_TEST_CASE(murmurhash3) #undef T } +BOOST_AUTO_TEST_CASE(siphash) +{ + CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x726fdb47dd0e0e31ull); + hasher.Write(0x0706050403020100ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x93f5f5799a932462ull); + hasher.Write(0x0F0E0D0C0B0A0908ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x3f2acc7f57c29bdbull); + hasher.Write(0x1716151413121110ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0xb8ad50c6f649af94ull); + hasher.Write(0x1F1E1D1C1B1A1918ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x7127512f72f27cceull); + hasher.Write(0x2726252423222120ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0x0e3ea96b5304a7d0ull); + hasher.Write(0x2F2E2D2C2B2A2928ULL); + BOOST_CHECK_EQUAL(hasher.Finalize(), 0xe612a3cb9ecba951ull); + + BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index 8de1b7c31419..ad6e374f6fe1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,10 +5,8 @@ #include "txdb.h" -#include "chain.h" #include "chainparams.h" #include "hash.h" -#include "validation.h" #include "pow.h" #include "uint256.h" @@ -53,7 +51,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CDBBatch batch(&db.GetObfuscateKey()); + CDBBatch batch(db); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -98,54 +96,52 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read(DB_LAST_BLOCK, nFile); } -bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { +CCoinsViewCursor *CCoinsViewDB::Cursor() const +{ + CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(&db)->NewIterator(), GetBestBlock()); /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COINS); + // Cache key of first record + i->pcursor->GetKey(i->keyTmp); + return i; +} - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - stats.hashBlock = GetBestBlock(); - ss << stats.hashBlock; - CAmount nTotalAmount = 0; - while (pcursor->Valid()) { - boost::this_thread::interruption_point(); - std::pair key; - CCoins coins; - if (pcursor->GetKey(key) && key.first == DB_COINS) { - if (pcursor->GetValue(coins)) { - stats.nTransactions++; - for (unsigned int i=0; iGetValueSize(); - ss << VARINT(0); - } else { - return error("CCoinsViewDB::GetStats() : unable to read value"); - } - } else { - break; - } - pcursor->Next(); - } - { - LOCK(cs_main); - stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; +bool CCoinsViewDBCursor::GetKey(uint256 &key) const +{ + // Return cached key + if (keyTmp.first == DB_COINS) { + key = keyTmp.second; + return true; } - stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; - return true; + return false; +} + +bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +{ + return pcursor->GetValue(coins); +} + +unsigned int CCoinsViewDBCursor::GetValueSize() const +{ + return pcursor->GetValueSize(); +} + +bool CCoinsViewDBCursor::Valid() const +{ + return keyTmp.first == DB_COINS; +} + +void CCoinsViewDBCursor::Next() +{ + pcursor->Next(); + if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -161,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); @@ -172,7 +168,7 @@ bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) } bool CBlockTreeDB::UpdateSpentIndex(const std::vector >&vect) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { if (it->second.IsNull()) { batch.Erase(make_pair(DB_SPENTINDEX, it->first)); @@ -184,7 +180,7 @@ bool CBlockTreeDB::UpdateSpentIndex(const std::vector >&vect) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { if (it->second.IsNull()) { batch.Erase(make_pair(DB_ADDRESSUNSPENTINDEX, it->first)); @@ -222,14 +218,14 @@ bool CBlockTreeDB::ReadAddressUnspentIndex(uint160 addressHash, int type, } bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); return WriteBatch(batch); } bool CBlockTreeDB::EraseAddressIndex(const std::vector >&vect) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Erase(make_pair(DB_ADDRESSINDEX, it->first)); return WriteBatch(batch); @@ -270,7 +266,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, } bool CBlockTreeDB::WriteTimestampIndex(const CTimestampIndexKey ×tampIndex) { - CDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(*this); batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), 0); return WriteBatch(batch); } @@ -307,7 +303,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } -bool CBlockTreeDB::LoadBlockIndexGuts() +bool CBlockTreeDB::LoadBlockIndexGuts(boost::function insertBlockIndex) { boost::scoped_ptr pcursor(NewIterator()); @@ -321,8 +317,8 @@ bool CBlockTreeDB::LoadBlockIndexGuts() CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object - CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); - pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); + CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash()); + pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; diff --git a/src/txdb.h b/src/txdb.h index 14d501278f23..cbfd336a0a0c 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -8,32 +8,59 @@ #include "coins.h" #include "dbwrapper.h" +#include "chain.h" +#include "spentindex.h" #include #include #include #include -class CBlockFileInfo; +#include + class CBlockIndex; -struct CDiskTxPos; -struct CAddressUnspentKey; -struct CAddressUnspentValue; -struct CAddressIndexKey; -struct CAddressIndexIteratorKey; -struct CAddressIndexIteratorHeightKey; -struct CTimestampIndexKey; -struct CTimestampIndexIteratorKey; -struct CSpentIndexKey; -struct CSpentIndexValue; +class CCoinsViewDBCursor; class uint256; //! -dbcache default (MiB) -static const int64_t nDefaultDbCache = 100; -//! max. -dbcache in (MiB) +static const int64_t nDefaultDbCache = 300; +//! max. -dbcache (MiB) static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; -//! min. -dbcache in (MiB) +//! min. -dbcache (MiB) static const int64_t nMinDbCache = 4; +//! Max memory allocated to block tree DB specific cache, if no -txindex (MiB) +static const int64_t nMaxBlockDBCache = 2; +//! Max memory allocated to block tree DB specific cache, if -txindex (MiB) +// Unlike for the UTXO database, for the txindex scenario the leveldb cache make +// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991 +static const int64_t nMaxBlockDBAndTxIndexCache = 1024; +//! Max memory allocated to coin DB specific cache (MiB) +static const int64_t nMaxCoinsDBCache = 8; + +struct CDiskTxPos : public CDiskBlockPos +{ + unsigned int nTxOffset; // after header + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(*(CDiskBlockPos*)this); + READWRITE(VARINT(nTxOffset)); + } + + CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + } + + CDiskTxPos() { + SetNull(); + } + + void SetNull() { + CDiskBlockPos::SetNull(); + nTxOffset = 0; + } +}; /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB : public CCoinsView @@ -47,7 +74,29 @@ class CCoinsViewDB : public CCoinsView bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - bool GetStats(CCoinsStats &stats) const; + CCoinsViewCursor *Cursor() const; +}; + +/** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ +class CCoinsViewDBCursor: public CCoinsViewCursor +{ +public: + ~CCoinsViewDBCursor() {} + + bool GetKey(uint256 &key) const; + bool GetValue(CCoins &coins) const; + unsigned int GetValueSize() const; + + bool Valid() const; + void Next(); + +private: + CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): + CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} + boost::scoped_ptr pcursor; + std::pair keyTmp; + + friend class CCoinsViewDB; }; /** Access to the block database (blocks/index/) */ @@ -80,7 +129,7 @@ class CBlockTreeDB : public CDBWrapper bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(); + bool LoadBlockIndexGuts(boost::function insertBlockIndex); }; #endif // BITCOIN_TXDB_H diff --git a/src/uint256.cpp b/src/uint256.cpp index c58c88bf4a8f..f22ddcd1effc 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -80,67 +80,3 @@ template std::string base_blob<256>::GetHex() const; template std::string base_blob<256>::ToString() const; template void base_blob<256>::SetHex(const char*); template void base_blob<256>::SetHex(const std::string&); - -static void inline HashMix(uint32_t& a, uint32_t& b, uint32_t& c) -{ - // Taken from lookup3, by Bob Jenkins. - a -= c; - a ^= ((c << 4) | (c >> 28)); - c += b; - b -= a; - b ^= ((a << 6) | (a >> 26)); - a += c; - c -= b; - c ^= ((b << 8) | (b >> 24)); - b += a; - a -= c; - a ^= ((c << 16) | (c >> 16)); - c += b; - b -= a; - b ^= ((a << 19) | (a >> 13)); - a += c; - c -= b; - c ^= ((b << 4) | (b >> 28)); - b += a; -} - -static void inline HashFinal(uint32_t& a, uint32_t& b, uint32_t& c) -{ - // Taken from lookup3, by Bob Jenkins. - c ^= b; - c -= ((b << 14) | (b >> 18)); - a ^= c; - a -= ((c << 11) | (c >> 21)); - b ^= a; - b -= ((a << 25) | (a >> 7)); - c ^= b; - c -= ((b << 16) | (b >> 16)); - a ^= c; - a -= ((c << 4) | (c >> 28)); - b ^= a; - b -= ((a << 14) | (a >> 18)); - c ^= b; - c -= ((b << 24) | (b >> 8)); -} - -uint64_t uint256::GetHash(const uint256& salt) const -{ - uint32_t a, b, c; - const uint32_t *pn = (const uint32_t*)data; - const uint32_t *salt_pn = (const uint32_t*)salt.data; - a = b = c = 0xdeadbeef + WIDTH; - - a += pn[0] ^ salt_pn[0]; - b += pn[1] ^ salt_pn[1]; - c += pn[2] ^ salt_pn[2]; - HashMix(a, b, c); - a += pn[3] ^ salt_pn[3]; - b += pn[4] ^ salt_pn[4]; - c += pn[5] ^ salt_pn[5]; - HashMix(a, b, c); - a += pn[6] ^ salt_pn[6]; - b += pn[7] ^ salt_pn[7]; - HashFinal(a, b, c); - - return ((((uint64_t)b) << 32) | c); -} diff --git a/src/uint256.h b/src/uint256.h index f3a73e385e1b..ecaa23070795 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -82,6 +82,19 @@ class base_blob return sizeof(data); } + uint64_t GetUint64(int pos) const + { + const uint8_t* ptr = data + pos * 8; + return ((uint64_t)ptr[0]) | \ + ((uint64_t)ptr[1]) << 8 | \ + ((uint64_t)ptr[2]) << 16 | \ + ((uint64_t)ptr[3]) << 24 | \ + ((uint64_t)ptr[4]) << 32 | \ + ((uint64_t)ptr[5]) << 40 | \ + ((uint64_t)ptr[6]) << 48 | \ + ((uint64_t)ptr[7]) << 56; + } + template void Serialize(Stream& s, int nType, int nVersion) const { @@ -126,11 +139,6 @@ class uint256 : public base_blob<256> { { return ReadLE64(data); } - - /** A more secure, salted hash function. - * @note This hash is not stable between little and big endian. - */ - uint64_t GetHash(const uint256& salt) const; }; /* uint256 from const char *. diff --git a/src/validation.cpp b/src/validation.cpp index c2ec90860f9c..b8222ebc5e6a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1445,17 +1445,9 @@ void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCach undo.nVersion = coins->nVersion; } } - // add outputs - inputs.ModifyNewCoins(tx.GetHash())->FromTx(tx, nHeight); - } - else { - // add outputs for coinbase tx - // In this case call the full ModifyCoins which will do a database - // lookup to be sure the coins do not already exist otherwise we do not - // know whether to mark them fresh or not. We want the duplicate coinbases - // before BIP30 to still be properly overwritten. - inputs.ModifyCoins(tx.GetHash())->FromTx(tx, nHeight); } + // add outputs + inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); } void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, int nHeight) @@ -3709,7 +3701,7 @@ CBlockIndex * InsertBlockIndex(uint256 hash) bool static LoadBlockIndexDB() { const CChainParams& chainparams = Params(); - if (!pblocktree->LoadBlockIndexGuts()) + if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; boost::this_thread::interruption_point(); diff --git a/src/validation.h b/src/validation.h index 42250903a651..80c8f284b7c7 100644 --- a/src/validation.h +++ b/src/validation.h @@ -310,306 +310,7 @@ std::string FormatStateMessage(const CValidationState &state); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); -struct CTimestampIndexIteratorKey { - unsigned int timestamp; - - size_t GetSerializeSize(int nType, int nVersion) const { - return 4; - } - template - void Serialize(Stream& s, int nType, int nVersion) const { - ser_writedata32be(s, timestamp); - } - template - void Unserialize(Stream& s, int nType, int nVersion) { - timestamp = ser_readdata32be(s); - } - - CTimestampIndexIteratorKey(unsigned int time) { - timestamp = time; - } - - CTimestampIndexIteratorKey() { - SetNull(); - } - - void SetNull() { - timestamp = 0; - } -}; - -struct CTimestampIndexKey { - unsigned int timestamp; - uint256 blockHash; - - size_t GetSerializeSize(int nType, int nVersion) const { - return 36; - } - template - void Serialize(Stream& s, int nType, int nVersion) const { - ser_writedata32be(s, timestamp); - blockHash.Serialize(s, nType, nVersion); - } - template - void Unserialize(Stream& s, int nType, int nVersion) { - timestamp = ser_readdata32be(s); - blockHash.Unserialize(s, nType, nVersion); - } - - CTimestampIndexKey(unsigned int time, uint256 hash) { - timestamp = time; - blockHash = hash; - } - - CTimestampIndexKey() { - SetNull(); - } - - void SetNull() { - timestamp = 0; - blockHash.SetNull(); - } -}; - -struct CAddressUnspentKey { - unsigned int type; - uint160 hashBytes; - uint256 txhash; - size_t index; - - size_t GetSerializeSize(int nType, int nVersion) const { - return 57; - } - template - void Serialize(Stream& s, int nType, int nVersion) const { - ser_writedata8(s, type); - hashBytes.Serialize(s, nType, nVersion); - txhash.Serialize(s, nType, nVersion); - ser_writedata32(s, index); - } - template - void Unserialize(Stream& s, int nType, int nVersion) { - type = ser_readdata8(s); - hashBytes.Unserialize(s, nType, nVersion); - txhash.Unserialize(s, nType, nVersion); - index = ser_readdata32(s); - } - - CAddressUnspentKey(unsigned int addressType, uint160 addressHash, uint256 txid, size_t indexValue) { - type = addressType; - hashBytes = addressHash; - txhash = txid; - index = indexValue; - } - - CAddressUnspentKey() { - SetNull(); - } - - void SetNull() { - type = 0; - hashBytes.SetNull(); - txhash.SetNull(); - index = 0; - } -}; - -struct CAddressUnspentValue { - CAmount satoshis; - CScript script; - int blockHeight; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(satoshis); - READWRITE(*(CScriptBase*)(&script)); - READWRITE(blockHeight); - } - - CAddressUnspentValue(CAmount sats, CScript scriptPubKey, int height) { - satoshis = sats; - script = scriptPubKey; - blockHeight = height; - } - - CAddressUnspentValue() { - SetNull(); - } - - void SetNull() { - satoshis = -1; - script.clear(); - blockHeight = 0; - } - - bool IsNull() const { - return (satoshis == -1); - } -}; - -struct CAddressIndexKey { - unsigned int type; - uint160 hashBytes; - int blockHeight; - unsigned int txindex; - uint256 txhash; - size_t index; - bool spending; - - size_t GetSerializeSize(int nType, int nVersion) const { - return 66; - } - template - void Serialize(Stream& s, int nType, int nVersion) const { - ser_writedata8(s, type); - hashBytes.Serialize(s, nType, nVersion); - // Heights are stored big-endian for key sorting in LevelDB - ser_writedata32be(s, blockHeight); - ser_writedata32be(s, txindex); - txhash.Serialize(s, nType, nVersion); - ser_writedata32(s, index); - char f = spending; - ser_writedata8(s, f); - } - template - void Unserialize(Stream& s, int nType, int nVersion) { - type = ser_readdata8(s); - hashBytes.Unserialize(s, nType, nVersion); - blockHeight = ser_readdata32be(s); - txindex = ser_readdata32be(s); - txhash.Unserialize(s, nType, nVersion); - index = ser_readdata32(s); - char f = ser_readdata8(s); - spending = f; - } - - CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t indexValue, bool isSpending) { - type = addressType; - hashBytes = addressHash; - blockHeight = height; - txindex = blockindex; - txhash = txid; - index = indexValue; - spending = isSpending; - } - - CAddressIndexKey() { - SetNull(); - } - - void SetNull() { - type = 0; - hashBytes.SetNull(); - blockHeight = 0; - txindex = 0; - txhash.SetNull(); - index = 0; - spending = false; - } - -}; - -struct CAddressIndexIteratorKey { - unsigned int type; - uint160 hashBytes; - - size_t GetSerializeSize(int nType, int nVersion) const { - return 21; - } - template - void Serialize(Stream& s, int nType, int nVersion) const { - ser_writedata8(s, type); - hashBytes.Serialize(s, nType, nVersion); - } - template - void Unserialize(Stream& s, int nType, int nVersion) { - type = ser_readdata8(s); - hashBytes.Unserialize(s, nType, nVersion); - } - - CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { - type = addressType; - hashBytes = addressHash; - } - - CAddressIndexIteratorKey() { - SetNull(); - } - - void SetNull() { - type = 0; - hashBytes.SetNull(); - } -}; - -struct CAddressIndexIteratorHeightKey { - unsigned int type; - uint160 hashBytes; - int blockHeight; - - size_t GetSerializeSize(int nType, int nVersion) const { - return 25; - } - template - void Serialize(Stream& s, int nType, int nVersion) const { - ser_writedata8(s, type); - hashBytes.Serialize(s, nType, nVersion); - ser_writedata32be(s, blockHeight); - } - template - void Unserialize(Stream& s, int nType, int nVersion) { - type = ser_readdata8(s); - hashBytes.Unserialize(s, nType, nVersion); - blockHeight = ser_readdata32be(s); - } - - CAddressIndexIteratorHeightKey(unsigned int addressType, uint160 addressHash, int height) { - type = addressType; - hashBytes = addressHash; - blockHeight = height; - } - - CAddressIndexIteratorHeightKey() { - SetNull(); - } - - void SetNull() { - type = 0; - hashBytes.SetNull(); - blockHeight = 0; - } -}; - -struct CDiskTxPos : public CDiskBlockPos -{ - unsigned int nTxOffset; // after header - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(*(CDiskBlockPos*)this); - READWRITE(VARINT(nTxOffset)); - } - - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - - CDiskTxPos() { - SetNull(); - } - - void SetNull() { - CDiskBlockPos::SetNull(); - nTxOffset = 0; - } -}; - - -/** +/** * Count ECDSA signature operations the old-fashioned (pre-0.6) way * @return number of sigops this transaction's outputs will produce when spent * @see CTransaction::FetchInputs @@ -752,61 +453,6 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); - -class CBlockFileInfo -{ -public: - unsigned int nBlocks; //! number of blocks stored in file - unsigned int nSize; //! number of used bytes of block file - unsigned int nUndoSize; //! number of used bytes in the undo file - unsigned int nHeightFirst; //! lowest height of block in file - unsigned int nHeightLast; //! highest height of block in file - uint64_t nTimeFirst; //! earliest time of block in file - uint64_t nTimeLast; //! latest time of block in file - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(VARINT(nBlocks)); - READWRITE(VARINT(nSize)); - READWRITE(VARINT(nUndoSize)); - READWRITE(VARINT(nHeightFirst)); - READWRITE(VARINT(nHeightLast)); - READWRITE(VARINT(nTimeFirst)); - READWRITE(VARINT(nTimeLast)); - } - - void SetNull() { - nBlocks = 0; - nSize = 0; - nUndoSize = 0; - nHeightFirst = 0; - nHeightLast = 0; - nTimeFirst = 0; - nTimeLast = 0; - } - - CBlockFileInfo() { - SetNull(); - } - - std::string ToString() const; - - /** update statistics (does not update nSize) */ - void AddBlock(unsigned int nHeightIn, uint64_t nTimeIn) { - if (nBlocks==0 || nHeightFirst > nHeightIn) - nHeightFirst = nHeightIn; - if (nBlocks==0 || nTimeFirst > nTimeIn) - nTimeFirst = nTimeIn; - nBlocks++; - if (nHeightIn > nHeightLast) - nHeightLast = nHeightIn; - if (nTimeIn > nTimeLast) - nTimeLast = nTimeIn; - } -}; - /** RAII wrapper for VerifyDB: Verify consistency of the block and coin databases */ class CVerifyDB { public: