diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 980f1e8f19cd..b9bf7bf46e54 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -12,7 +12,7 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ bdbpath=X bdb48path=X bdbdirlist= - for _vn in 4.8 48 4 5 ''; do + for _vn in 4.8 48 4 5 5.3 ''; do for _pfx in b lib ''; do bdbdirlist="$bdbdirlist ${_pfx}db${_vn}" done diff --git a/configure.ac b/configure.ac index c68d83079a78..a34f37cda753 100644 --- a/configure.ac +++ b/configure.ac @@ -178,6 +178,16 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=$enableval], [use_glibc_compat=no]) +AC_ARG_ENABLE([experimental-asm], + [AS_HELP_STRING([--enable-experimental-asm], + [Enable experimental assembly routines (default is no)])], + [experimental_asm=$enableval], + [experimental_asm=no]) + +if test "x$experimental_asm" = xyes; then + AC_DEFINE(EXPERIMENTAL_ASM, 1, [Define this symbol to build in experimental assembly routines]) +fi + AC_ARG_WITH([system-univalue], [AS_HELP_STRING([--with-system-univalue], [Build with system UniValue (default is no)])], @@ -1218,6 +1228,7 @@ AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_HWCRC32],[test x$enable_hwcrc32 = xyes]) +AM_CONDITIONAL([EXPERIMENTAL_ASM],[test x$experimental_asm = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) diff --git a/src/.clang-format b/src/.clang-format index f1113e635362..a8f45e4ca790 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -25,7 +25,6 @@ ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false -ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH, BOOST_REVERSE_FOREACH ] IndentCaseLabels: false IndentFunctionDeclarationAfterType: false IndentWidth: 4 diff --git a/src/Makefile.am b/src/Makefile.am index 4354f2d36f48..23af507b6e87 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -194,6 +194,7 @@ BITCOIN_CORE_H = \ pow.h \ protocol.h \ random.h \ + reverse_iterator.h \ reverselock.h \ rpc/blockchain.h \ rpc/client.h \ @@ -384,6 +385,10 @@ crypto_libdash_crypto_a_SOURCES = \ crypto/sha512.cpp \ crypto/sha512.h +if EXPERIMENTAL_ASM +crypto_libdash_crypto_a_SOURCES += crypto/sha256_sse4.cpp +endif + # x11 crypto_libdash_crypto_a_SOURCES += \ crypto/blake.c \ diff --git a/src/bench/bench_dash.cpp b/src/bench/bench_dash.cpp index 2cb94af3cff2..b5a899ad8d15 100644 --- a/src/bench/bench_dash.cpp +++ b/src/bench/bench_dash.cpp @@ -4,6 +4,7 @@ #include "bench.h" +#include "crypto/sha256.h" #include "key.h" #include "stacktraces.h" #include "validation.h" @@ -18,6 +19,8 @@ void CleanupBLSDkgTests(); int main(int argc, char** argv) { + SHA256AutoDetect(); + RegisterPrettySignalHandlers(); RegisterPrettyTerminateHander(); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 0e86568e5bf8..b6881cb0835c 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -5,7 +5,6 @@ #include "bench.h" #include "wallet/wallet.h" -#include #include static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector& vCoins) diff --git a/src/bloom.cpp b/src/bloom.cpp index 1a58b8714205..7ca1071234bc 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -18,7 +18,6 @@ #include #include -#include #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index 6f4f0f2825bc..42c55756a073 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -7,12 +7,12 @@ #include "chain.h" #include "chainparams.h" +#include "reverse_iterator.h" #include "validation.h" #include "uint256.h" #include -#include namespace Checkpoints { @@ -20,7 +20,7 @@ namespace Checkpoints { { const MapCheckpoints& checkpoints = data.mapCheckpoints; - BOOST_REVERSE_FOREACH(const MapCheckpoints::value_type& i, checkpoints) + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; BlockMap::const_iterator t = mapBlockIndex.find(hash); diff --git a/src/checkqueue.h b/src/checkqueue.h index d7b7b836dcf2..408e278d2162 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -10,7 +10,6 @@ #include #include -#include #include #include diff --git a/src/coins.cpp b/src/coins.cpp index 801b94d79cd8..a727ae714e69 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -9,7 +9,6 @@ #include "random.h" #include -#include bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } diff --git a/src/core_memusage.h b/src/core_memusage.h index a6781556843c..9dca695fe4b0 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -10,7 +10,7 @@ #include "memusage.h" static inline size_t RecursiveDynamicUsage(const CScript& script) { - return memusage::DynamicUsage(*static_cast(&script)); + return memusage::DynamicUsage(script); } static inline size_t RecursiveDynamicUsage(const COutPoint& out) { diff --git a/src/core_write.cpp b/src/core_write.cpp index edb60b55db55..87664f0ac948 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -22,7 +22,6 @@ #include "evo/specialtx.h" #include "llmq/quorums_commitment.h" -#include std::string FormatScript(const CScript& script) { diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 5b9f00a0a2dc..15d6db90c246 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -3,10 +3,21 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/sha256.h" - #include "crypto/common.h" +#include #include +#include + +#if defined(__x86_64__) || defined(__amd64__) +#if defined(EXPERIMENTAL_ASM) +#include +namespace sha256_sse4 +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} +#endif +#endif // Internal implementation code. namespace @@ -43,93 +54,142 @@ void inline Initialize(uint32_t* s) s[7] = 0x5be0cd19ul; } -/** Perform one SHA-256 transformation, processing a 64-byte chunk. */ -void Transform(uint32_t* s, const unsigned char* chunk) +/** Perform a number of SHA-256 transformations, processing 64-byte chunks. */ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) { - uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; - uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; - - Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = ReadBE32(chunk + 0)); - Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = ReadBE32(chunk + 4)); - Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = ReadBE32(chunk + 8)); - Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = ReadBE32(chunk + 12)); - Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = ReadBE32(chunk + 16)); - Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = ReadBE32(chunk + 20)); - Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = ReadBE32(chunk + 24)); - Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = ReadBE32(chunk + 28)); - Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = ReadBE32(chunk + 32)); - Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = ReadBE32(chunk + 36)); - Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = ReadBE32(chunk + 40)); - Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = ReadBE32(chunk + 44)); - Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = ReadBE32(chunk + 48)); - Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = ReadBE32(chunk + 52)); - Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = ReadBE32(chunk + 56)); - Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = ReadBE32(chunk + 60)); - - Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x240ca1cc, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x2de92c6f, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x4a7484aa, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x76f988da, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0x983e5152, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0xa831c66d, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0xb00327c8, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0xbf597fc7, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xd5a79147, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0x06ca6351, w14 += sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0x14292967, w15 += sigma1(w13) + w8 + sigma0(w0)); - - Round(a, b, c, d, e, f, g, h, 0x27b70a85, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0x2e1b2138, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x53380d13, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x650a7354, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x766a0abb, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x81c2c92e, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x92722c85, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0xa81a664b, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0xc24b8b70, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0xc76c51a3, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0xd192e819, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xd6990624, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0xf40e3585, w14 += sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0x106aa070, w15 += sigma1(w13) + w8 + sigma0(w0)); - - Round(a, b, c, d, e, f, g, h, 0x19a4c116, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0x1e376c08, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x2748774c, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x391c0cb3, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x682e6ff3, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0x748f82ee, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0x78a5636f, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0x84c87814, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0x8cc70208, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0x90befffa, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xa4506ceb, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, w14 + sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0xc67178f2, w15 + sigma1(w13) + w8 + sigma0(w0)); - - s[0] += a; - s[1] += b; - s[2] += c; - s[3] += d; - s[4] += e; - s[5] += f; - s[6] += g; - s[7] += h; + while (blocks--) { + uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; + uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = ReadBE32(chunk + 0)); + Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = ReadBE32(chunk + 4)); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = ReadBE32(chunk + 8)); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = ReadBE32(chunk + 12)); + Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = ReadBE32(chunk + 16)); + Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = ReadBE32(chunk + 20)); + Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = ReadBE32(chunk + 24)); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = ReadBE32(chunk + 28)); + Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = ReadBE32(chunk + 32)); + Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = ReadBE32(chunk + 36)); + Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = ReadBE32(chunk + 40)); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = ReadBE32(chunk + 44)); + Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = ReadBE32(chunk + 48)); + Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = ReadBE32(chunk + 52)); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = ReadBE32(chunk + 56)); + Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = ReadBE32(chunk + 60)); + + Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x240ca1cc, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x2de92c6f, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x4a7484aa, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x76f988da, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0x983e5152, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0xa831c66d, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0xb00327c8, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xd5a79147, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0x06ca6351, w14 += sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0x14292967, w15 += sigma1(w13) + w8 + sigma0(w0)); + + Round(a, b, c, d, e, f, g, h, 0x27b70a85, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x53380d13, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x650a7354, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x766a0abb, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x81c2c92e, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x92722c85, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0xa81a664b, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0xd192e819, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xd6990624, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0xf40e3585, w14 += sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0x106aa070, w15 += sigma1(w13) + w8 + sigma0(w0)); + + Round(a, b, c, d, e, f, g, h, 0x19a4c116, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0x1e376c08, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x2748774c, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0x748f82ee, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0x78a5636f, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0x84c87814, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0x8cc70208, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0x90befffa, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xa4506ceb, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, w14 + sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0xc67178f2, w15 + sigma1(w13) + w8 + sigma0(w0)); + + s[0] += a; + s[1] += b; + s[2] += c; + s[3] += d; + s[4] += e; + s[5] += f; + s[6] += g; + s[7] += h; + chunk += 64; + } } } // namespace sha256 + +typedef void (*TransformType)(uint32_t*, const unsigned char*, size_t); + +bool SelfTest(TransformType tr) { + static const unsigned char in1[65] = {0, 0x80}; + static const unsigned char in2[129] = { + 0, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 + }; + static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}; + static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul}; + static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul}; + uint32_t buf[8]; + memcpy(buf, init, sizeof(buf)); + // Process nothing, and check we remain in the initial state. + tr(buf, nullptr, 0); + if (memcmp(buf, init, sizeof(buf))) return false; + // Process the padded empty string (unaligned) + tr(buf, in1 + 1, 1); + if (memcmp(buf, out1, sizeof(buf))) return false; + // Process 64 spaces (unaligned) + memcpy(buf, init, sizeof(buf)); + tr(buf, in2 + 1, 2); + if (memcmp(buf, out2, sizeof(buf))) return false; + return true; +} + +TransformType Transform = sha256::Transform; + } // namespace +std::string SHA256AutoDetect() +{ +#if defined(EXPERIMENTAL_ASM) && (defined(__x86_64__) || defined(__amd64__)) + uint32_t eax, ebx, ecx, edx; + if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) { + Transform = sha256_sse4::Transform; + assert(SelfTest(Transform)); + return "sse4"; + } +#endif + + assert(SelfTest(Transform)); + return "standard"; +} ////// SHA-256 @@ -147,14 +207,14 @@ CSHA256& CSHA256::Write(const unsigned char* data, size_t len) memcpy(buf + bufsize, data, 64 - bufsize); bytes += 64 - bufsize; data += 64 - bufsize; - sha256::Transform(s, buf); + Transform(s, buf, 1); bufsize = 0; } - while (end >= data + 64) { - // Process full chunks directly from the source. - sha256::Transform(s, data); - bytes += 64; - data += 64; + if (end - data >= 64) { + size_t blocks = (end - data) / 64; + Transform(s, data, blocks); + data += 64 * blocks; + bytes += 64 * blocks; } if (end > data) { // Fill the buffer with what remains. diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index 5b15b6a233b1..51608af9c7d1 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -7,6 +7,7 @@ #include #include +#include /** A hasher class for SHA-256. */ class CSHA256 @@ -25,4 +26,9 @@ class CSHA256 CSHA256& Reset(); }; +/** Autodetect the best available SHA256 implementation. + * Returns the name of the implementation. + */ +std::string SHA256AutoDetect(); + #endif // BITCOIN_CRYPTO_SHA256_H diff --git a/src/crypto/sha256_sse4.cpp b/src/crypto/sha256_sse4.cpp new file mode 100644 index 000000000000..89f529a3abcf --- /dev/null +++ b/src/crypto/sha256_sse4.cpp @@ -0,0 +1,1506 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// This is a translation to GCC extended asm syntax from YASM code by Intel +// (available at the bottom of this file). + +#include +#include + +#if defined(__x86_64__) || defined(__amd64__) + +namespace sha256_sse4 +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + static const uint32_t K256 alignas(16) [] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + }; + static const uint32_t FLIP_MASK alignas(16) [] = {0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f}; + static const uint32_t SHUF_00BA alignas(16) [] = {0x03020100, 0x0b0a0908, 0xffffffff, 0xffffffff}; + static const uint32_t SHUF_DC00 alignas(16) [] = {0xffffffff, 0xffffffff, 0x03020100, 0x0b0a0908}; + uint32_t a, b, c, d, f, g, h, y0, y1, y2; + uint64_t tbl; + uint64_t inp_end, inp; + uint32_t xfer alignas(16) [4]; + + __asm__ __volatile__( + "shl $0x6,%2;" + "je Ldone_hash_%=;" + "add %1,%2;" + "mov %2,%14;" + "mov (%0),%3;" + "mov 0x4(%0),%4;" + "mov 0x8(%0),%5;" + "mov 0xc(%0),%6;" + "mov 0x10(%0),%k2;" + "mov 0x14(%0),%7;" + "mov 0x18(%0),%8;" + "mov 0x1c(%0),%9;" + "movdqa %18,%%xmm12;" + "movdqa %19,%%xmm10;" + "movdqa %20,%%xmm11;" + + "Lloop0_%=:" + "lea %17,%13;" + "movdqu (%1),%%xmm4;" + "pshufb %%xmm12,%%xmm4;" + "movdqu 0x10(%1),%%xmm5;" + "pshufb %%xmm12,%%xmm5;" + "movdqu 0x20(%1),%%xmm6;" + "pshufb %%xmm12,%%xmm6;" + "movdqu 0x30(%1),%%xmm7;" + "pshufb %%xmm12,%%xmm7;" + "mov %1,%15;" + "mov $3,%1;" + + "Lloop1_%=:" + "movdqa 0x0(%13),%%xmm9;" + "paddd %%xmm4,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm7,%%xmm0;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "palignr $0x4,%%xmm6,%%xmm0;" + "ror $0x9,%11;" + "xor %k2,%10;" + "mov %7,%12;" + "ror $0x5,%10;" + "movdqa %%xmm5,%%xmm1;" + "xor %3,%11;" + "xor %8,%12;" + "paddd %%xmm4,%%xmm0;" + "xor %k2,%10;" + "and %k2,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm4,%%xmm1;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "pslld $0x19,%%xmm1;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "psrld $0x7,%%xmm2;" + "and %4,%10;" + "add %11,%9;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%9;" + "movdqa %%xmm3,%%xmm2;" + "mov %6,%10;" + "mov %9,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %6,%10;" + "mov %k2,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %6,%10;" + "and %6,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %9,%11;" + "xor %7,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "pxor %%xmm8,%%xmm1;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "pshufd $0xfa,%%xmm7,%%xmm2;" + "and %3,%10;" + "add %11,%8;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%8;" + "movdqa %%xmm2,%%xmm3;" + "mov %5,%10;" + "mov %8,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %k2,%12;" + "psrlq $0x13,%%xmm3;" + "xor %5,%10;" + "and %5,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %8,%11;" + "xor %k2,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "paddd %%xmm8,%%xmm0;" + "and %9,%10;" + "add %11,%7;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%7;" + "movdqa %%xmm2,%%xmm3;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "movdqa %%xmm2,%%xmm4;" + "ror $0x9,%11;" + "xor %4,%10;" + "mov %5,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %7,%11;" + "xor %6,%12;" + "psrlq $0x13,%%xmm3;" + "xor %4,%10;" + "and %4,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm4;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm4;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "pshufb %%xmm11,%%xmm4;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "paddd %%xmm0,%%xmm4;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "movdqa 0x10(%13),%%xmm9;" + "paddd %%xmm5,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm4,%%xmm0;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "palignr $0x4,%%xmm7,%%xmm0;" + "ror $0x9,%11;" + "xor %3,%10;" + "mov %4,%12;" + "ror $0x5,%10;" + "movdqa %%xmm6,%%xmm1;" + "xor %k2,%11;" + "xor %5,%12;" + "paddd %%xmm5,%%xmm0;" + "xor %3,%10;" + "and %3,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm5,%%xmm1;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "pslld $0x19,%%xmm1;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "psrld $0x7,%%xmm2;" + "and %7,%10;" + "add %11,%6;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%6;" + "movdqa %%xmm3,%%xmm2;" + "mov %9,%10;" + "mov %6,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %9,%10;" + "mov %3,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %9,%10;" + "and %9,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %6,%11;" + "xor %4,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "pxor %%xmm8,%%xmm1;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "pshufd $0xfa,%%xmm4,%%xmm2;" + "and %k2,%10;" + "add %11,%5;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%5;" + "movdqa %%xmm2,%%xmm3;" + "mov %8,%10;" + "mov %5,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %3,%12;" + "psrlq $0x13,%%xmm3;" + "xor %8,%10;" + "and %8,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %5,%11;" + "xor %3,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "paddd %%xmm8,%%xmm0;" + "and %6,%10;" + "add %11,%4;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%4;" + "movdqa %%xmm2,%%xmm3;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "movdqa %%xmm2,%%xmm5;" + "ror $0x9,%11;" + "xor %7,%10;" + "mov %8,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %4,%11;" + "xor %9,%12;" + "psrlq $0x13,%%xmm3;" + "xor %7,%10;" + "and %7,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm5;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm5;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "pshufb %%xmm11,%%xmm5;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "paddd %%xmm0,%%xmm5;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "movdqa 0x20(%13),%%xmm9;" + "paddd %%xmm6,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm5,%%xmm0;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "palignr $0x4,%%xmm4,%%xmm0;" + "ror $0x9,%11;" + "xor %k2,%10;" + "mov %7,%12;" + "ror $0x5,%10;" + "movdqa %%xmm7,%%xmm1;" + "xor %3,%11;" + "xor %8,%12;" + "paddd %%xmm6,%%xmm0;" + "xor %k2,%10;" + "and %k2,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm6,%%xmm1;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "pslld $0x19,%%xmm1;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "psrld $0x7,%%xmm2;" + "and %4,%10;" + "add %11,%9;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%9;" + "movdqa %%xmm3,%%xmm2;" + "mov %6,%10;" + "mov %9,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %6,%10;" + "mov %k2,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %6,%10;" + "and %6,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %9,%11;" + "xor %7,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "pxor %%xmm8,%%xmm1;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "pshufd $0xfa,%%xmm5,%%xmm2;" + "and %3,%10;" + "add %11,%8;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%8;" + "movdqa %%xmm2,%%xmm3;" + "mov %5,%10;" + "mov %8,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %k2,%12;" + "psrlq $0x13,%%xmm3;" + "xor %5,%10;" + "and %5,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %8,%11;" + "xor %k2,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "paddd %%xmm8,%%xmm0;" + "and %9,%10;" + "add %11,%7;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%7;" + "movdqa %%xmm2,%%xmm3;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "movdqa %%xmm2,%%xmm6;" + "ror $0x9,%11;" + "xor %4,%10;" + "mov %5,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %7,%11;" + "xor %6,%12;" + "psrlq $0x13,%%xmm3;" + "xor %4,%10;" + "and %4,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm6;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm6;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "pshufb %%xmm11,%%xmm6;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "paddd %%xmm0,%%xmm6;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "movdqa 0x30(%13),%%xmm9;" + "paddd %%xmm7,%%xmm9;" + "movdqa %%xmm9,%16;" + "add $0x40,%13;" + "movdqa %%xmm6,%%xmm0;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "palignr $0x4,%%xmm5,%%xmm0;" + "ror $0x9,%11;" + "xor %3,%10;" + "mov %4,%12;" + "ror $0x5,%10;" + "movdqa %%xmm4,%%xmm1;" + "xor %k2,%11;" + "xor %5,%12;" + "paddd %%xmm7,%%xmm0;" + "xor %3,%10;" + "and %3,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm7,%%xmm1;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "pslld $0x19,%%xmm1;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "psrld $0x7,%%xmm2;" + "and %7,%10;" + "add %11,%6;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%6;" + "movdqa %%xmm3,%%xmm2;" + "mov %9,%10;" + "mov %6,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %9,%10;" + "mov %3,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %9,%10;" + "and %9,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %6,%11;" + "xor %4,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "pxor %%xmm8,%%xmm1;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "pshufd $0xfa,%%xmm6,%%xmm2;" + "and %k2,%10;" + "add %11,%5;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%5;" + "movdqa %%xmm2,%%xmm3;" + "mov %8,%10;" + "mov %5,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %3,%12;" + "psrlq $0x13,%%xmm3;" + "xor %8,%10;" + "and %8,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %5,%11;" + "xor %3,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "paddd %%xmm8,%%xmm0;" + "and %6,%10;" + "add %11,%4;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%4;" + "movdqa %%xmm2,%%xmm3;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "movdqa %%xmm2,%%xmm7;" + "ror $0x9,%11;" + "xor %7,%10;" + "mov %8,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %4,%11;" + "xor %9,%12;" + "psrlq $0x13,%%xmm3;" + "xor %7,%10;" + "and %7,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm7;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm7;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "pshufb %%xmm11,%%xmm7;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "paddd %%xmm0,%%xmm7;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "sub $0x1,%1;" + "jne Lloop1_%=;" + "mov $0x2,%1;" + + "Lloop2_%=:" + "paddd 0x0(%13),%%xmm4;" + "movdqa %%xmm4,%16;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "xor %k2,%10;" + "ror $0x9,%11;" + "mov %7,%12;" + "xor %3,%11;" + "ror $0x5,%10;" + "xor %8,%12;" + "xor %k2,%10;" + "ror $0xb,%11;" + "and %k2,%12;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add %16,%12;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "and %4,%10;" + "add %11,%9;" + "or %12,%10;" + "add %10,%9;" + "mov %6,%10;" + "ror $0xe,%10;" + "mov %9,%11;" + "xor %6,%10;" + "ror $0x9,%11;" + "mov %k2,%12;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "xor %6,%10;" + "ror $0xb,%11;" + "and %6,%12;" + "xor %9,%11;" + "ror $0x6,%10;" + "xor %7,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 4+%16,%12;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "and %3,%10;" + "add %11,%8;" + "or %12,%10;" + "add %10,%8;" + "mov %5,%10;" + "ror $0xe,%10;" + "mov %8,%11;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "xor %k2,%12;" + "xor %5,%10;" + "ror $0xb,%11;" + "and %5,%12;" + "xor %8,%11;" + "ror $0x6,%10;" + "xor %k2,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "and %9,%10;" + "add %11,%7;" + "or %12,%10;" + "add %10,%7;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "xor %4,%10;" + "ror $0x9,%11;" + "mov %5,%12;" + "xor %7,%11;" + "ror $0x5,%10;" + "xor %6,%12;" + "xor %4,%10;" + "ror $0xb,%11;" + "and %4,%12;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 12+%16,%12;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "paddd 0x10(%13),%%xmm5;" + "movdqa %%xmm5,%16;" + "add $0x20,%13;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "xor %3,%10;" + "ror $0x9,%11;" + "mov %4,%12;" + "xor %k2,%11;" + "ror $0x5,%10;" + "xor %5,%12;" + "xor %3,%10;" + "ror $0xb,%11;" + "and %3,%12;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add %16,%12;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "and %7,%10;" + "add %11,%6;" + "or %12,%10;" + "add %10,%6;" + "mov %9,%10;" + "ror $0xe,%10;" + "mov %6,%11;" + "xor %9,%10;" + "ror $0x9,%11;" + "mov %3,%12;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "xor %9,%10;" + "ror $0xb,%11;" + "and %9,%12;" + "xor %6,%11;" + "ror $0x6,%10;" + "xor %4,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 4+%16,%12;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "and %k2,%10;" + "add %11,%5;" + "or %12,%10;" + "add %10,%5;" + "mov %8,%10;" + "ror $0xe,%10;" + "mov %5,%11;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "xor %3,%12;" + "xor %8,%10;" + "ror $0xb,%11;" + "and %8,%12;" + "xor %5,%11;" + "ror $0x6,%10;" + "xor %3,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "and %6,%10;" + "add %11,%4;" + "or %12,%10;" + "add %10,%4;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "xor %7,%10;" + "ror $0x9,%11;" + "mov %8,%12;" + "xor %4,%11;" + "ror $0x5,%10;" + "xor %9,%12;" + "xor %7,%10;" + "ror $0xb,%11;" + "and %7,%12;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 12+%16,%12;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "movdqa %%xmm6,%%xmm4;" + "movdqa %%xmm7,%%xmm5;" + "sub $0x1,%1;" + "jne Lloop2_%=;" + "add (%0),%3;" + "mov %3,(%0);" + "add 0x4(%0),%4;" + "mov %4,0x4(%0);" + "add 0x8(%0),%5;" + "mov %5,0x8(%0);" + "add 0xc(%0),%6;" + "mov %6,0xc(%0);" + "add 0x10(%0),%k2;" + "mov %k2,0x10(%0);" + "add 0x14(%0),%7;" + "mov %7,0x14(%0);" + "add 0x18(%0),%8;" + "mov %8,0x18(%0);" + "add 0x1c(%0),%9;" + "mov %9,0x1c(%0);" + "mov %15,%1;" + "add $0x40,%1;" + "cmp %14,%1;" + "jne Lloop0_%=;" + + "Ldone_hash_%=:" + + : "+r"(s), "+r"(chunk), "+r"(blocks), "=r"(a), "=r"(b), "=r"(c), "=r"(d), /* e = chunk */ "=r"(f), "=r"(g), "=r"(h), "=r"(y0), "=r"(y1), "=r"(y2), "=r"(tbl), "+m"(inp_end), "+m"(inp), "+m"(xfer) + : "m"(K256), "m"(FLIP_MASK), "m"(SHUF_00BA), "m"(SHUF_DC00) + : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12" + ); +} +} + +/* +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright (c) 2012, Intel Corporation +; +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are +; met: +; +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in the +; documentation and/or other materials provided with the +; distribution. +; +; * Neither the name of the Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived from +; this software without specific prior written permission. +; +; +; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY +; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +; PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR +; CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Example YASM command lines: +; Windows: yasm -Xvc -f x64 -rnasm -pnasm -o sha256_sse4.obj -g cv8 sha256_sse4.asm +; Linux: yasm -f x64 -f elf64 -X gnu -g dwarf2 -D LINUX -o sha256_sse4.o sha256_sse4.asm +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; This code is described in an Intel White-Paper: +; "Fast SHA-256 Implementations on Intel Architecture Processors" +; +; To find it, surf to http://www.intel.com/p/en_US/embedded +; and search for that title. +; The paper is expected to be released roughly at the end of April, 2012 +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; This code schedules 1 blocks at a time, with 4 lanes per block +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define MOVDQ movdqu ;; assume buffers not aligned + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros + +; addm [mem], reg +; Add reg to mem using reg-mem add and store +%macro addm 2 + add %2, %1 + mov %1, %2 +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask +; Load xmm with mem and byte swap each dword +%macro COPY_XMM_AND_BSWAP 3 + MOVDQ %1, %2 + pshufb %1, %3 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define X0 xmm4 +%define X1 xmm5 +%define X2 xmm6 +%define X3 xmm7 + +%define XTMP0 xmm0 +%define XTMP1 xmm1 +%define XTMP2 xmm2 +%define XTMP3 xmm3 +%define XTMP4 xmm8 +%define XFER xmm9 + +%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA +%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 +%define BYTE_FLIP_MASK xmm12 + +%ifdef LINUX +%define NUM_BLKS rdx ; 3rd arg +%define CTX rsi ; 2nd arg +%define INP rdi ; 1st arg + +%define SRND rdi ; clobbers INP +%define c ecx +%define d r8d +%define e edx +%else +%define NUM_BLKS r8 ; 3rd arg +%define CTX rdx ; 2nd arg +%define INP rcx ; 1st arg + +%define SRND rcx ; clobbers INP +%define c edi +%define d esi +%define e r8d + +%endif +%define TBL rbp +%define a eax +%define b ebx + +%define f r9d +%define g r10d +%define h r11d + +%define y0 r13d +%define y1 r14d +%define y2 r15d + + + +_INP_END_SIZE equ 8 +_INP_SIZE equ 8 +_XFER_SIZE equ 8 +%ifdef LINUX +_XMM_SAVE_SIZE equ 0 +%else +_XMM_SAVE_SIZE equ 7*16 +%endif +; STACK_SIZE plus pushes must be an odd multiple of 8 +_ALIGN_SIZE equ 8 + +_INP_END equ 0 +_INP equ _INP_END + _INP_END_SIZE +_XFER equ _INP + _INP_SIZE +_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE +STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE + +; rotate_Xs +; Rotate values of symbols X0...X3 +%macro rotate_Xs 0 +%xdefine X_ X0 +%xdefine X0 X1 +%xdefine X1 X2 +%xdefine X2 X3 +%xdefine X3 X_ +%endm + +; ROTATE_ARGS +; Rotate values of symbols a...h +%macro ROTATE_ARGS 0 +%xdefine TMP_ h +%xdefine h g +%xdefine g f +%xdefine f e +%xdefine e d +%xdefine d c +%xdefine c b +%xdefine b a +%xdefine a TMP_ +%endm + +%macro FOUR_ROUNDS_AND_SCHED 0 + ;; compute s0 four at a time and s1 two at a time + ;; compute W[-16] + W[-7] 4 at a time + movdqa XTMP0, X3 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + palignr XTMP0, X2, 4 ; XTMP0 = W[-7] + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + movdqa XTMP1, X1 + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + ;; compute s0 + palignr XTMP1, X0, 4 ; XTMP1 = W[-15] + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH + movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pslld XTMP1, (32-7) + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + psrld XTMP2, 7 + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] + mov y0, e ; y0 = e + mov y1, a ; y1 = a + movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] + ror y0, (25-11) ; y0 = e >> (25-11) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y1, (22-13) ; y1 = a >> (22-13) + pslld XTMP3, (32-18) + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + psrld XTMP2, 18 + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP1, XTMP3 + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pxor XTMP1, XTMP4 ; XTMP1 = s0 + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + ;; compute low s1 + pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} + mov y0, e ; y0 = e + mov y1, a ; y1 = a + ror y0, (25-11) ; y0 = e >> (25-11) + movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP2, XTMP3 + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH + pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + ;; compute high s1 + pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + pxor XTMP2, XTMP3 + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH + pxor X0, XTMP2 ; X0 = s1 {xDxC} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS +rotate_Xs +%endm + +;; input is [rsp + _XFER + %1 * 4] +%macro DO_ROUND 1 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + and y2, e ; y2 = (f^g)&e + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha256_sse4(void *input_data, UINT32 digest[8], UINT64 num_blks) +;; arg 1 : pointer to input data +;; arg 2 : pointer to digest +;; arg 3 : Num blocks +section .text +global sha256_sse4 +align 32 +sha256_sse4: + push rbx +%ifndef LINUX + push rsi + push rdi +%endif + push rbp + push r13 + push r14 + push r15 + + sub rsp,STACK_SIZE +%ifndef LINUX + movdqa [rsp + _XMM_SAVE + 0*16],xmm6 + movdqa [rsp + _XMM_SAVE + 1*16],xmm7 + movdqa [rsp + _XMM_SAVE + 2*16],xmm8 + movdqa [rsp + _XMM_SAVE + 3*16],xmm9 + movdqa [rsp + _XMM_SAVE + 4*16],xmm10 + movdqa [rsp + _XMM_SAVE + 5*16],xmm11 + movdqa [rsp + _XMM_SAVE + 6*16],xmm12 +%endif + + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + add NUM_BLKS, INP ; pointer to end of data + mov [rsp + _INP_END], NUM_BLKS + + ;; load initial digest + mov a,[4*0 + CTX] + mov b,[4*1 + CTX] + mov c,[4*2 + CTX] + mov d,[4*3 + CTX] + mov e,[4*4 + CTX] + mov f,[4*5 + CTX] + mov g,[4*6 + CTX] + mov h,[4*7 + CTX] + + movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] + movdqa SHUF_00BA, [_SHUF_00BA wrt rip] + movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] + +loop0: + lea TBL,[K256 wrt rip] + + ;; byte swap first 16 dwords + COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK + + mov [rsp + _INP], INP + + ;; schedule 48 input dwords, by doing 3 rounds of 16 each + mov SRND, 3 +align 16 +loop1: + movdqa XFER, [TBL + 0*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 1*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 2*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 3*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + add TBL, 4*16 + FOUR_ROUNDS_AND_SCHED + + sub SRND, 1 + jne loop1 + + mov SRND, 2 +loop2: + paddd X0, [TBL + 0*16] + movdqa [rsp + _XFER], X0 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + paddd X1, [TBL + 1*16] + movdqa [rsp + _XFER], X1 + add TBL, 2*16 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + movdqa X0, X2 + movdqa X1, X3 + + sub SRND, 1 + jne loop2 + + addm [4*0 + CTX],a + addm [4*1 + CTX],b + addm [4*2 + CTX],c + addm [4*3 + CTX],d + addm [4*4 + CTX],e + addm [4*5 + CTX],f + addm [4*6 + CTX],g + addm [4*7 + CTX],h + + mov INP, [rsp + _INP] + add INP, 64 + cmp INP, [rsp + _INP_END] + jne loop0 + +done_hash: +%ifndef LINUX + movdqa xmm6,[rsp + _XMM_SAVE + 0*16] + movdqa xmm7,[rsp + _XMM_SAVE + 1*16] + movdqa xmm8,[rsp + _XMM_SAVE + 2*16] + movdqa xmm9,[rsp + _XMM_SAVE + 3*16] + movdqa xmm10,[rsp + _XMM_SAVE + 4*16] + movdqa xmm11,[rsp + _XMM_SAVE + 5*16] + movdqa xmm12,[rsp + _XMM_SAVE + 6*16] +%endif + + add rsp, STACK_SIZE + + pop r15 + pop r14 + pop r13 + pop rbp +%ifndef LINUX + pop rdi + pop rsi +%endif + pop rbx + + ret + + +section .data +align 64 +K256: + dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203 + +; shuffle xBxA -> 00BA +_SHUF_00BA: ddq 0xFFFFFFFFFFFFFFFF0b0a090803020100 + +; shuffle xDxC -> DC00 +_SHUF_DC00: ddq 0x0b0a090803020100FFFFFFFFFFFFFFFF +*/ + +#endif diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 36a1fd651e5d..2104ac75785b 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -18,7 +18,6 @@ #include #include // boost::trim -#include /** WWW-Authenticate to present with 401 Unauthorized response */ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; diff --git a/src/init.cpp b/src/init.cpp index 35dd2cc58fbf..fa8aebda5906 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -270,6 +270,7 @@ void PrepareShutdown() if (fFeeEstimatesInitialized) { + ::feeEstimator.FlushUnconfirmed(::mempool); fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) @@ -1441,6 +1442,8 @@ bool AppInitSanityChecks() // ********************************************************* Step 4: sanity checks // Initialize elliptic curve code + std::string sha256_algo = SHA256AutoDetect(); + LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); ECC_Start(); globalVerifyHandle.reset(new ECCVerifyHandle()); diff --git a/src/memusage.h b/src/memusage.h index d73b1f937d51..8db0fd606706 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -15,7 +15,6 @@ #include #include -#include namespace memusage { diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index b2b003f666ff..fd856fc14811 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -71,6 +71,9 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, const std::set& txids) } uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::vector &vTxid) { + //we can never have zero txs in a merkle block, we always need the coinbase tx + //if we do not have this assert, we can hit a memory access violation when indexing into vTxid + assert(vTxid.size() != 0); if (height == 0) { // hash at height 0 is the txids themself return vTxid[pos]; diff --git a/src/merkleblock.h b/src/merkleblock.h index 68fac71a938d..8a974d4aaaea 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -121,6 +121,8 @@ class CPartialMerkleTree /** * Used to relay blocks as header + vector * to filtered nodes. + * + * NOTE: The class assumes that the given CBlock has *at least* 1 transaction. If the CBlock has 0 txs, it will hit an assertion. */ class CMerkleBlock { diff --git a/src/net.h b/src/net.h index 9683184c7544..2f7a545b8084 100644 --- a/src/net.h +++ b/src/net.h @@ -37,7 +37,6 @@ #include #endif -#include #include // "Optimistic send" was introduced in the beginning of the Bitcoin project. I assume this was done because it was diff --git a/src/net_processing.cpp b/src/net_processing.cpp index addf4d2807f9..0afc0e8e7bca 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -22,6 +22,7 @@ #include "primitives/block.h" #include "primitives/transaction.h" #include "random.h" +#include "reverse_iterator.h" #include "tinyformat.h" #include "txmempool.h" #include "ui_interface.h" @@ -900,7 +901,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB // Relay inventory, but don't relay old inventory during initial block download. connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { - BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { + for (const uint256& hash : reverse_iterate(vHashes)) { pnode->PushBlockHash(hash); } } @@ -2672,7 +2673,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } else { std::vector vGetData; // Download as much as possible, from earliest to latest. - BOOST_REVERSE_FOREACH(const CBlockIndex *pindex, vToFetch) { + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { // Can't download any more from this peer break; diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index a0c37a70d6fa..b8dede4f838a 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -13,6 +13,8 @@ #include "txmempool.h" #include "util.h" +static constexpr double INF_FEERATE = 1e99; + /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to their @@ -25,51 +27,55 @@ class TxConfirmStats { private: //Define the buckets we will group transactions into - std::vector buckets; // The upper-bound of the range for the bucket (inclusive) - std::map bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + const std::vector& buckets; // The upper-bound of the range for the bucket (inclusive) + const std::map& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket // For each bucket X: // Count the total # of txs in each bucket // Track the historical moving average of this total over blocks std::vector txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector curBlockTxCt; // Count the total # of txs confirmed within Y blocks in each bucket // Track the historical moving average of theses totals over blocks - std::vector > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector > curBlockConf; // curBlockConf[Y][X] + std::vector> confAvg; // confAvg[Y][X] + + // Track moving avg of txs which have been evicted from the mempool + // after failing to be confirmed within Y blocks + std::vector> failAvg; // failAvg[Y][X] // Sum the total feerate of all tx's in each bucket // Track the historical moving average of this total over blocks std::vector avg; - // and calculate the total for the current block to update the moving average - std::vector curBlockVal; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X // Combine the total value with the tx counts to calculate the avg feerate per bucket double decay; + // Resolution (# of blocks) with which confirmations are tracked + unsigned int scale; + // Mempool counts of outstanding transactions // For each bucket X, track the number of transactions in the mempool // that are unconfirmed for each possible confirmation value Y std::vector > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket + // transactions still unconfirmed after GetMaxConfirms for each bucket std::vector oldUnconfTxs; + void resizeInMemoryCounters(size_t newbuckets); + public: /** * Create new TxConfirmStats. This is called by BlockPolicyEstimator's * constructor with default values. * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track + * @param maxPeriods max number of periods to track * @param decay how much to decay the historical moving average per block */ - TxConfirmStats(const std::vector& defaultBuckets, unsigned int maxConfirms, double decay); + TxConfirmStats(const std::vector& defaultBuckets, const std::map& defaultBucketMap, + unsigned int maxPeriods, double decay, unsigned int scale); - /** Clear the state of the curBlock variables to start counting for the new block */ + /** Roll the circular buffer for unconfirmed txs*/ void ClearCurrent(unsigned int nBlockHeight); /** @@ -85,7 +91,7 @@ class TxConfirmStats /** Remove a transaction from mempool tracking stats*/ void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); + unsigned int bucketIndex, bool inBlock); /** Update our estimates by decaying our historical moving average and updating with the data gathered from the current block */ @@ -103,10 +109,11 @@ class TxConfirmStats * @param nBlockHeight the current block height */ double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight) const; + double minSuccess, bool requireGreater, unsigned int nBlockHeight, + EstimationResult *result = nullptr) const; /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() const { return confAvg.size(); } + unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } /** Write state of estimation data to a file*/ void Write(CAutoFile& fileout) const; @@ -115,44 +122,47 @@ class TxConfirmStats * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(CAutoFile& filein); + void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); }; TxConfirmStats::TxConfirmStats(const std::vector& defaultBuckets, - unsigned int maxConfirms, double _decay) + const std::map& defaultBucketMap, + unsigned int maxPeriods, double _decay, unsigned int _scale) + : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; - for (unsigned int i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { + scale = _scale; + confAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); + } + failAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { + failAvg[i].resize(buckets.size()); } - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); + + resizeInMemoryCounters(buckets.size()); } -// Zero out the data for the current block +void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) { + // newbuckets must be passed in because the buckets referred to during Read have not been updated yet. + unconfTxs.resize(GetMaxConfirms()); + for (unsigned int i = 0; i < unconfTxs.size(); i++) { + unconfTxs[i].resize(newbuckets); + } + oldUnconfTxs.resize(newbuckets); +} + +// Roll the unconfirmed txs circular buffer void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (unsigned int j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; - for (unsigned int i = 0; i < curBlockConf.size(); i++) - curBlockConf[i][j] = 0; - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; } } @@ -162,33 +172,38 @@ void TxConfirmStats::Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + scale - 1)/scale; unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; + for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) { + confAvg[i - 1][bucketindex]++; } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; + txCtAvg[bucketindex]++; + avg[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + confAvg[i][j] = confAvg[i][j] * decay; + for (unsigned int i = 0; i < failAvg.size(); i++) + failAvg[i][j] = failAvg[i][j] * decay; + avg[j] = avg[j] * decay; + txCtAvg[j] = txCtAvg[j] * decay; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) const + unsigned int nBlockHeight, EstimationResult *result) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + scale - 1)/scale; int maxbucketindex = buckets.size() - 1; @@ -211,12 +226,21 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket; + EstimatorBucket failBucket; // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; + nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; + failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; @@ -225,24 +249,41 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { + if (passing == true) { + // First time we hit a failure record the failed bucket + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters else { + failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; + passing = true; + passBucket.withinTarget = nConf; nConf = 0; + passBucket.totalConfirmed = totalNum; totalNum = 0; + passBucket.inMempool = extraNum; + passBucket.leftMempool = failNum; + failNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + newBucketRange = true; } } } @@ -254,8 +295,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int minBucket = std::min(bestNearBucket, bestFarBucket); + unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket); for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } @@ -269,83 +310,109 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, break; } } + + passBucket.start = minBucket ? buckets[minBucket-1] : 0; + passBucket.end = buckets[maxBucket]; } - LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], - 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) { + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + } + LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d %s%.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + confTarget, requireGreater ? ">" : "<", 100.0 * successBreakPoint, decay, + median, passBucket.start, passBucket.end, + 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool), + passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool, + failBucket.start, failBucket.end, + 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool), + failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool); + + + if (result) { + result->pass = passBucket; + result->fail = failBucket; + result->decay = decay; + result->scale = scale; + } return median; } void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; - fileout << buckets; + fileout << scale; fileout << avg; fileout << txCtAvg; fileout << confAvg; + fileout << failAvg; } -void TxConfirmStats::Read(CAutoFile& filein) +void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) { - // Read data file into temporary variables and do some very basic sanity checking - std::vector fileBuckets; - std::vector fileAvg; - std::vector > fileConfAvg; - std::vector fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - filein >> fileAvg; - if (fileAvg.size() != numBuckets) + // Read data file and do some very basic sanity checking + // buckets and bucketMap are not updated yet, so don't access them + // If there is a read failure, we'll just discard this entire object anyway + size_t maxConfirms, maxPeriods; + + // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor + if (nFileVersion >= 140100) { + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + } + + filein >> avg; + if (avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) + } + filein >> txCtAvg; + if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week - throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); + filein >> confAvg; + maxPeriods = confAvg.size(); + maxConfirms = scale * maxPeriods; - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (confAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); + if (nFileVersion >= 140100) { + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + } else { + failAvg.resize(confAvg.size()); + for (unsigned int i = 0; i < failAvg.size(); i++) { + failAvg[i].resize(numBuckets); + } } - oldUnconfTxs.resize(buckets.size()); - for (unsigned int i = 0; i < buckets.size(); i++) - bucketMap[buckets[i]] = i; + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + resizeInMemoryCounters(numBuckets); LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); @@ -359,7 +426,7 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) return bucketindex; } -void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; @@ -387,6 +454,12 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe blockIndex, bucketindex); } } + if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + unsigned int periodsAgo = blocksAgo / scale; + for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { + failAvg[i][bucketindex]++; + } + } } // This function is called from CTxMemPool::removeUnchecked to ensure @@ -394,12 +467,14 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // tracked. Txs that were part of a block have already been removed in // processBlockTx to ensure they are never double tracked, but it is // of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) +bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { LOCK(cs_feeEstimator); std::map::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); mapMemPoolTxs.erase(hash); return true; } else { @@ -408,21 +483,28 @@ bool CBlockPolicyEstimator::removeTx(uint256 hash) } CBlockPolicyEstimator::CBlockPolicyEstimator() - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) + : nBestSeenHeight(0), firstRecordedHeight(0), historicalFirst(0), historicalBest(0), trackedTxs(0), untrackedTxs(0) { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); - minTrackedFee = CFeeRate(MIN_BUCKET_FEERATE); - std::vector vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); + size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { + buckets.push_back(bucketBoundary); + bucketMap[bucketBoundary] = bucketIndex; } - vfeelist.push_back(INF_FEERATE); - feeStats = new TxConfirmStats(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + buckets.push_back(INF_FEERATE); + bucketMap[INF_FEERATE] = bucketIndex; + assert(bucketMap.size() == buckets.size()); + + feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); + shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); + longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); } CBlockPolicyEstimator::~CBlockPolicyEstimator() { delete feeStats; + delete shortStats; + delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) @@ -455,12 +537,17 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = bucketIndex; + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex2); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex3); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash())) { + if (!removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -480,6 +567,8 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } @@ -501,21 +590,32 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, // of unconfirmed txs to remove from tracking. nBestSeenHeight = nBlockHeight; - // Clear the current block state and update unconfirmed circular buffer + // Update unconfirmed circular buffer feeStats->ClearCurrent(nBlockHeight); + shortStats->ClearCurrent(nBlockHeight); + longStats->ClearCurrent(nBlockHeight); + + // Decay all exponential averages + feeStats->UpdateMovingAverages(); + shortStats->UpdateMovingAverages(); + longStats->UpdateMovingAverages(); unsigned int countedTxs = 0; - // Repopulate the current block states + // Update averages with data points from current block for (const auto& entry : entries) { if (processBlockTx(nBlockHeight, entry)) countedTxs++; } - // Update all exponential averages with the current block state - feeStats->UpdateMovingAverages(); + if (firstRecordedHeight == 0 && countedTxs > 0) { + firstRecordedHeight = nBestSeenHeight; + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight); + } - LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); + + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", + countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; untrackedTxs = 0; @@ -523,13 +623,44 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget <= 1) + return CFeeRate(0); + + return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE); +} + +CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const +{ + TxConfirmStats* stats; + double sufficientTxs = SUFFICIENT_FEETXS; + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + stats = shortStats; + sufficientTxs = SUFFICIENT_TXS_SHORT; + break; + } + case FeeEstimateHorizon::MED_HALFLIFE: { + stats = feeStats; + break; + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + stats = longStats; + break; + } + default: { + return CFeeRate(0); + } + } + LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms()) + return CFeeRate(0); + if (successThreshold > 1) return CFeeRate(0); - double median = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, nBestSeenHeight, result); if (median < 0) return CFeeRate(0); @@ -537,31 +668,148 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const +unsigned int CBlockPolicyEstimator::BlockSpan() const +{ + if (firstRecordedHeight == 0) return 0; + assert(nBestSeenHeight >= firstRecordedHeight); + + return nBestSeenHeight - firstRecordedHeight; +} + +unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const +{ + if (historicalFirst == 0) return 0; + assert(historicalBest >= historicalFirst); + + if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0; + + return historicalBest - historicalFirst; +} + +unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const +{ + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2); +} + +/** Return a fee estimate at the required successThreshold from the shortest + * time horizon which tracks confirmations up to the desired target. If + * checkShorterHorizon is requested, also allow short time horizon estimates + * for a lower target to reduce the given answer */ +double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const +{ + double estimate = -1; + if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) { + // Find estimate from shortest time horizon possible + if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon + estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + } + else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon + estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + else { // long horizon + estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + if (checkShorterHorizon) { + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > feeStats->GetMaxConfirms()) { + double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + estimate = medMax; + } + if (confTarget > shortStats->GetMaxConfirms()) { + double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + estimate = shortMax; + } + } + } + return estimate; +} + +/** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + * at 2 * target for any longer time horizons. + */ +double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget) const +{ + double estimate = -1; + if (doubleTarget <= shortStats->GetMaxConfirms()) { + estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + } + if (doubleTarget <= feeStats->GetMaxConfirms()) { + double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + if (longEstimate > estimate) { + estimate = longEstimate; + } + } + return estimate; +} + +/** estimateSmartFee returns the max of the feerates calculated with a 60% + * threshold required at target / 2, an 85% threshold required at target and a + * 95% threshold required at 2 * target. Each calculation is performed at the + * shortest time horizon which tracks the required target. Conservative + * estimates, however, required the 95% threshold at 2 * target be met for any + * longer time horizons also. + */ +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; double median = -1; - { LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) return CFeeRate(0); // It's not possible to get reasonable estimates for confTarget of 1 if (confTarget == 1) confTarget = 2; - while (median < 0 && (unsigned int)confTarget <= feeStats->GetMaxConfirms()) { - median = feeStats->EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + unsigned int maxUsableEstimate = MaxUsableEstimate(); + if (maxUsableEstimate <= 1) + return CFeeRate(0); + + if ((unsigned int)confTarget > maxUsableEstimate) { + confTarget = maxUsableEstimate; + } + + assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + /** true is passed to estimateCombined fee for target/2 and target so + * that we check the max confirms for shorter time horizons as well. + * This is necessary to preserve monotonically increasing estimates. + * For non-conservative estimates we do the same thing for 2*target, but + * for conservative estimates we want to skip these shorter horizons + * checks for 2*target because we are taking the max over all time + * horizons so we already have monotonically increasing estimates and + * the purpose of conservative estimates is not to let short term + * fluctuations lower our estimates by too much. + */ + double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true); + double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true); + double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative); + median = halfEst; + if (actualEst > median) { + median = actualEst; + } + if (doubleEst > median) { + median = doubleEst; + } + + if (conservative || median == -1) { + double consEst = estimateConservativeFee(2 * confTarget); + if (consEst > median) { + median = consEst; + } } } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; + *answerFoundAtTarget = confTarget; // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); @@ -574,6 +822,7 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } + bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { try { @@ -581,7 +830,16 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const fileout << 140100; // version required to read: 0.14.1 or later fileout << CLIENT_VERSION; // version that wrote the file fileout << nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()/2) { + fileout << firstRecordedHeight << nBestSeenHeight; + } + else { + fileout << historicalFirst << historicalBest; + } + fileout << buckets; feeStats->Write(fileout); + shortStats->Write(fileout); + longStats->Write(fileout); } catch (const std::exception&) { LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); @@ -594,19 +852,95 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) { try { LOCK(cs_feeEstimator); - int nVersionRequired, nVersionThatWrote, nFileBestSeenHeight; + int nVersionRequired, nVersionThatWrote; + unsigned int nFileBestSeenHeight, nFileHistoricalFirst, nFileHistoricalBest; filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + + // Read fee estimates file into temporary variables so existing data + // structures aren't corrupted if there is an exception. filein >> nFileBestSeenHeight; - feeStats->Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 120300 then another TxConfirmStats (for priority) follows but can be ignored. + + if (nVersionThatWrote < 140100) { + // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. + // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor + double tempDecay; + filein >> tempDecay; + if (tempDecay <= 0 || tempDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + std::vector tempBuckets; + filein >> tempBuckets; + size_t tempNum = tempBuckets.size(); + if (tempNum <= 1 || tempNum > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::map tempMap; + + std::unique_ptr tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); + tempFeeStats->Read(filein, nVersionThatWrote, tempNum); + // if nVersionThatWrote < 120300 then another TxConfirmStats (for priority) follows but can be ignored. + + tempMap.clear(); + for (unsigned int i = 0; i < tempBuckets.size(); i++) { + tempMap[tempBuckets[i]] = i; + } + } + else { // nVersionThatWrote >= 140100 + filein >> nFileHistoricalFirst >> nFileHistoricalBest; + if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { + throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); + } + std::vector fileBuckets; + filein >> fileBuckets; + size_t numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::unique_ptr fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + std::unique_ptr fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + std::unique_ptr fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); + fileShortStats->Read(filein, nVersionThatWrote, numBuckets); + fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + + // Fee estimates file parsed correctly + // Copy buckets from file and refresh our bucketmap + buckets = fileBuckets; + bucketMap.clear(); + for (unsigned int i = 0; i < buckets.size(); i++) { + bucketMap[buckets[i]] = i; + } + + // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap + delete feeStats; + delete shortStats; + delete longStats; + feeStats = fileFeeStats.release(); + shortStats = fileShortStats.release(); + longStats = fileLongStats.release(); + + nBestSeenHeight = nFileBestSeenHeight; + historicalFirst = nFileHistoricalFirst; + historicalBest = nFileHistoricalBest; + } } - catch (const std::exception&) { - LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal)\n"); + catch (const std::exception& e) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); return false; } return true; } +void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { + int64_t startclear = GetTimeMicros(); + std::vector txids; + pool.queryHashes(txids); + LOCK(cs_feeEstimator); + for (auto& txid : txids) { + removeTx(txid, false); + } + int64_t endclear = GetTimeMicros(); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %ld micros\n",txids.size(), endclear - startclear); +} diff --git a/src/policy/fees.h b/src/policy/fees.h index ce087c238e3e..d2ee816a1998 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -42,53 +42,57 @@ class TxConfirmStats; * within your desired 5 blocks. * * Here is a brief description of the implementation: - * When a transaction enters the mempool, we - * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of feerate - * paid in each bucket. Then we calculate how many blocks Y it took each - * transaction to be mined and we track an array of counters in each bucket - * for how long it to took transactions to get confirmed from 1 to a max of 25 - * and we increment all the counters from Y up to 25. This is because for any - * number Z>=Y the transaction was successfully mined within Z blocks. We - * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given feerate - * bucket and the total number that were confirmed in each number 1-25 blocks - * or less for any bucket. We save this history by keeping an exponentially - * decaying moving average of each one of these stats. Furthermore we also - * keep track of the number unmined (in mempool) transactions in each bucket - * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that feerate bucket when calculating - * an estimate for any number of confirmations below the number of blocks - * they've been outstanding. + * When a transaction enters the mempool, we track the height of the block chain + * at entry. All further calculations are conducted only on this set of "seen" + * transactions. Whenever a block comes in, we count the number of transactions + * in each bucket and the total amount of feerate paid in each bucket. Then we + * calculate how many blocks Y it took each transaction to be mined. We convert + * from a number of blocks to a number of periods Y' each encompassing "scale" + * blocks. This is is tracked in 3 different data sets each up to a maximum + * number of periods. Within each data set we have an array of counters in each + * feerate bucket and we increment all the counters from Y' up to max periods + * representing that a tx was successfully confirmed in less than or equal to + * that many periods. We want to save a history of this information, so at any + * time we have a counter of the total number of transactions that happened in a + * given feerate bucket and the total number that were confirmed in each of the + * periods or less for any bucket. We save this history by keeping an + * exponentially decaying moving average of each one of these stats. This is + * done for a different decay in each of the 3 data sets to keep relevant data + * from different time horizons. Furthermore we also keep track of the number + * unmined (in mempool or left mempool without being included in a block) + * transactions in each bucket and for how many blocks they have been + * outstanding and use both of these numbers to increase the number of transactions + * we've seen in that feerate bucket when calculating an estimate for any number + * of confirmations below the number of blocks they've been outstanding. */ -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 14.4 hours */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ -static const double SUFFICIENT_FEETXS = 1; +/* Identifier for each of the 3 different TxConfirmStats which will track + * history over different time horizons. */ +enum FeeEstimateHorizon { + SHORT_HALFLIFE = 0, + MED_HALFLIFE = 1, + LONG_HALFLIFE = 2 +}; -// Minimum and Maximum values for tracking feerates -// The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we -// might ever want to track. Historically this has been 1000 since it was -// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it -// invalidates old estimates files. So leave it at 1000 unless it becomes -// necessary to lower it, and then lower it substantially. -static constexpr double MIN_BUCKET_FEERATE = 1000; -static const double MAX_BUCKET_FEERATE = 1e7; -static const double INF_FEERATE = MAX_MONEY; +/* Used to return detailed information about a feerate bucket */ +struct EstimatorBucket +{ + double start = -1; + double end = -1; + double withinTarget = 0; + double totalConfirmed = 0; + double inMempool = 0; + double leftMempool = 0; +}; -// We have to lump transactions into buckets based on feerate, but we want to be able -// to give accurate estimates over a large range of potential feerates -// Therefore it makes sense to exponentially space the buckets -/** Spacing of FeeRate buckets */ -static const double FEE_SPACING = 1.1; +/* Used to return detailed information about a fee estimate calculation */ +struct EstimationResult +{ + EstimatorBucket pass; + EstimatorBucket fail; + double decay; + unsigned int scale; +}; /** * We want to be able to estimate feerates that are needed on tx's to be included in @@ -97,6 +101,55 @@ static const double FEE_SPACING = 1.1; */ class CBlockPolicyEstimator { +private: + /** Track confirm delays up to 12 blocks for short horizon */ + static constexpr unsigned int SHORT_BLOCK_PERIODS = 12; + static constexpr unsigned int SHORT_SCALE = 1; + /** Track confirm delays up to 48 blocks for medium horizon */ + static constexpr unsigned int MED_BLOCK_PERIODS = 24; + static constexpr unsigned int MED_SCALE = 2; + /** Track confirm delays up to 1008 blocks for long horizon */ + static constexpr unsigned int LONG_BLOCK_PERIODS = 42; + static constexpr unsigned int LONG_SCALE = 24; + /** Historical estimates that are older than this aren't valid */ + static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; + + /** Decay of .962 is a half-life of 18 blocks or about 45 minutes */ + static constexpr double SHORT_DECAY = .962; + /** Decay of .998 is a half-life of 144 blocks or about 6 hours */ + static constexpr double MED_DECAY = .9952; + /** Decay of .9995 is a half-life of 1008 blocks or about 2 days */ + static constexpr double LONG_DECAY = .99931; + + /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ + static constexpr double HALF_SUCCESS_PCT = .6; + /** Require greater than 85% of X feerate transactions to be confirmed within Y blocks*/ + static constexpr double SUCCESS_PCT = .85; + /** Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks*/ + static constexpr double DOUBLE_SUCCESS_PCT = .95; + + /** Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance */ + static constexpr double SUFFICIENT_FEETXS = 0.1; + /** Require an avg of 0.5 tx when using short decay since there are fewer blocks considered*/ + static constexpr double SUFFICIENT_TXS_SHORT = 0.5; + + /** Minimum and Maximum values for tracking feerates + * The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we + * might ever want to track. Historically this has been 1000 since it was + * inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + * invalidates old estimates files. So leave it at 1000 unless it becomes + * necessary to lower it, and then lower it substantially. + */ + static constexpr double MIN_BUCKET_FEERATE = 1000; + static constexpr double MAX_BUCKET_FEERATE = 1e7; + + /** Spacing of FeeRate buckets + * We have to lump transactions into buckets based on feerate, but we want to be able + * to give accurate estimates over a large range of potential feerates + * Therefore it makes sense to exponentially space the buckets + */ + static constexpr double FEE_SPACING = 1.05; + public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); @@ -110,16 +163,23 @@ class CBlockPolicyEstimator void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); + bool removeTx(uint256 hash, bool inBlock); - /** Return a feerate estimate */ + /** DEPRECATED. Return a feerate estimate */ CFeeRate estimateFee(int confTarget) const; - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + /** Estimate feerate needed to get be included in a block within confTarget + * blocks. If no answer can be given at confTarget, return an estimate at + * the closest target where one can be given. 'conservative' estimates are + * valid over longer time horizons also. + */ + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative = true) const; + + /** Return a specific fee estimate calculation with a given success + * threshold and time horizon, and optionally return detailed data about + * calculation */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const; + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; /** Write estimation data to a file */ bool Write(CAutoFile& fileout) const; @@ -127,9 +187,15 @@ class CBlockPolicyEstimator /** Read estimation data from a file */ bool Read(CAutoFile& filein); + /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ + void FlushUnconfirmed(CTxMemPool& pool); + private: - CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main unsigned int nBestSeenHeight; + unsigned int firstRecordedHeight; + unsigned int historicalFirst; + unsigned int historicalBest; + struct TxStatsInfo { unsigned int blockHeight; @@ -142,14 +208,30 @@ class CBlockPolicyEstimator /** Classes to track historical data on transaction confirmations */ TxConfirmStats* feeStats; + TxConfirmStats* shortStats; + TxConfirmStats* longStats; unsigned int trackedTxs; unsigned int untrackedTxs; + std::vector buckets; // The upper-bound of the range for the bucket (inclusive) + std::map bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + mutable CCriticalSection cs_feeEstimator; /** Process a transaction confirmed in a block*/ bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + /** Helper for estimateSmartFee */ + double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const; + /** Helper for estimateSmartFee */ + double estimateConservativeFee(unsigned int doubleTarget) const; + /** Number of blocks of data recorded while fee estimates have been running */ + unsigned int BlockSpan() const; + /** Number of blocks of recorded fee estimate data represented in saved data file */ + unsigned int HistoricalBlockSpan() const; + /** Calculation of highest target that reasonable estimate can be provided for */ + unsigned int MaxUsableEstimate() const; }; + #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index cf07968e15df..864c88445b96 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -13,7 +13,6 @@ #include "util.h" #include "utilstrencodings.h" -#include CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) { diff --git a/src/prevector.h b/src/prevector.h index c5f84f1b7a74..c72546fea429 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -132,7 +132,7 @@ class prevector { typedef const T* pointer; typedef const T& reference; typedef std::bidirectional_iterator_tag iterator_category; - const_reverse_iterator(T* ptr_) : ptr(ptr_) {} + const_reverse_iterator(const T* ptr_) : ptr(ptr_) {} const_reverse_iterator(reverse_iterator x) : ptr(&(*x)) {} const T& operator*() const { return *ptr; } const T* operator->() const { return ptr; } diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index bdcd5d5dc304..e10cbde8a2b1 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -114,7 +114,7 @@ class CTxIn template inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(prevout); - READWRITE(*(CScriptBase*)(&scriptSig)); + READWRITE(scriptSig); READWRITE(nSequence); } @@ -160,7 +160,7 @@ class CTxOut template inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(nValue); - READWRITE(*(CScriptBase*)(&scriptPubKey)); + READWRITE(scriptPubKey); } void SetNull() diff --git a/src/protocol.h b/src/protocol.h index 9f363fe058a2..e7ee254f0a18 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -163,7 +163,7 @@ extern const char *PONG; /** * The notfound message is a reply to a getdata message which requested an * object the receiving node does not have available for relay. - * @ince protocol version 70001. + * @since protocol version 70001. * @see https://bitcoin.org/en/developer-reference#notfound */ extern const char *NOTFOUND; diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 9c8b03992a4c..2a6fb40cae8d 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -11,7 +11,6 @@ #include "base58.h" #include "wallet/wallet.h" -#include #include #include diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index df2a88aaf553..81d9c5f6f6df 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -19,6 +19,7 @@ #include "txmempool.h" #include "ui_interface.h" #include "util.h" +#include "warnings.h" #include "masternode/masternode-sync.h" #include "privatesend/privatesend.h" diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index e1850983535a..3558861e3554 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -11,7 +11,6 @@ #include "clientversion.h" #include "streams.h" -#include RecentRequestsTableModel::RecentRequestsTableModel(CWallet *wallet, WalletModel *parent) : QAbstractTableModel(parent), walletModel(parent) diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0a17126766a5..e74bb95bfbc5 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -57,8 +57,8 @@ const TrafficGraphData::GraphRange INITIAL_TRAFFIC_GRAPH_SETTING = TrafficGraphD // Repair parameters const QString SALVAGEWALLET("-salvagewallet"); const QString RESCAN("-rescan"); -const QString ZAPTXES1("-zapwallettxes=1"); -const QString ZAPTXES2("-zapwallettxes=2"); +const QString ZAPTXES1("-zapwallettxes=1 -persistmempool=0"); +const QString ZAPTXES2("-zapwallettxes=2 -persistmempool=0"); const QString UPGRADEWALLET("-upgradewallet"); const QString REINDEX("-reindex"); diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 7fde2aa4349e..6e1903dc5161 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -15,7 +15,6 @@ #include -#include /* Return positive answer if transaction should be shown in list. */ diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index b2ff459a3bc3..9ec2dacb4e1d 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -34,7 +34,6 @@ #include #include -#include WalletModel::WalletModel(const PlatformStyle *platformStyle, CWallet *_wallet, OptionsModel *_optionsModel, QObject *parent) : QObject(parent), wallet(_wallet), optionsModel(_optionsModel), addressTableModel(0), diff --git a/src/random.cpp b/src/random.cpp index 41146451c48e..c6a9906a1e65 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -186,6 +186,7 @@ void GetDevURandom(unsigned char *ent32) do { ssize_t n = read(f, ent32 + have, NUM_OS_RANDOM_BYTES - have); if (n <= 0 || n + have > NUM_OS_RANDOM_BYTES) { + close(f); RandFailure(); } have += n; @@ -226,10 +227,12 @@ void GetOSRand(unsigned char *ent32) RandFailure(); } } -#elif defined(HAVE_GETENTROPY) +#elif defined(HAVE_GETENTROPY) && defined(__OpenBSD__) /* On OpenBSD this can return up to 256 bytes of entropy, will return an * error if more are requested. * The call cannot return less than the requested number of bytes. + getentropy is explicitly limited to openbsd here, as a similar (but not + the same) function may exist on other platforms via glibc. */ if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); diff --git a/src/reverse_iterator.h b/src/reverse_iterator.h new file mode 100644 index 000000000000..409f895ce062 --- /dev/null +++ b/src/reverse_iterator.h @@ -0,0 +1,39 @@ +// Taken from https://gist.github.com/arvidsson/7231973 + +#ifndef BITCOIN_REVERSE_ITERATOR_HPP +#define BITCOIN_REVERSE_ITERATOR_HPP + +/** + * Template used for reverse iteration in C++11 range-based for loops. + * + * std::vector v = {1, 2, 3, 4, 5}; + * for (auto x : reverse_iterate(v)) + * std::cout << x << " "; + */ + +template +class reverse_range +{ + T &x; + +public: + reverse_range(T &x) : x(x) {} + + auto begin() const -> decltype(this->x.rbegin()) + { + return x.rbegin(); + } + + auto end() const -> decltype(this->x.rend()) + { + return x.rend(); + } +}; + +template +reverse_range reverse_iterate(T &x) +{ + return reverse_range(x); +} + +#endif // BITCOIN_REVERSE_ITERATOR_HPP diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 71f5d00af0d2..c58f8e8a1588 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1036,7 +1036,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, stats.nTransactions++; for (const auto output : outputs) { ss << VARINT(output.first + 1); - ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << output.second.out.scriptPubKey; ss << VARINT(output.second.out.nValue); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 10ca11922859..40772d08e6ba 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -137,6 +137,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getrawmempool", 0, "verbose" }, { "estimatefee", 0, "nblocks" }, { "estimatesmartfee", 0, "nblocks" }, + { "estimatesmartfee", 1, "conservative" }, + { "estimaterawfee", 0, "nblocks" }, + { "estimaterawfee", 1, "threshold" }, + { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "fee_delta" }, { "setban", 2, "bantime" }, { "setban", 3, "absolute" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 7bdec5bb29e4..05478c1cc71a 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -25,6 +25,7 @@ #include "util.h" #include "utilstrencodings.h" #include "validationinterface.h" +#include "warnings.h" #include "governance/governance-classes.h" #include "masternode/masternode-payments.h" @@ -824,6 +825,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "estimatefee nblocks\n" + "\nDEPRECATED. Please use estimatesmartfee for more intelligent estimates." "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks.\n" "\nArguments:\n" @@ -854,15 +856,18 @@ UniValue estimatefee(const JSONRPCRequest& request) UniValue estimatesmartfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "estimatesmartfee nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "estimatesmartfee nblocks (conservative)\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid.\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric)\n" + "2. conservative (bool, optional, default=true) Whether to return a more conservative estimate which\n" + " also satisfies a longer history. A conservative estimate potentially returns a higher\n" + " feerate and is more likely to be sufficient for the desired target, but is not as\n" + " responsive to short term drops in the prevailing fee market\n" "\nResult:\n" "{\n" " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in " + CURRENCY_UNIT + ")\n" @@ -879,15 +884,101 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM}); int nBlocks = request.params[0].get_int(); + bool conservative = true; + if (request.params.size() > 1 && !request.params[1].isNull()) { + RPCTypeCheckArgument(request.params[1], UniValue::VBOOL); + conservative = request.params[1].get_bool(); + } UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool, conservative); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; } +UniValue estimaterawfee(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + throw std::runtime_error( + "estimaterawfee nblocks (threshold horizon)\n" + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + " implementation of fee estimation. The parameters it can be called with\n" + " and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within nblocks blocks if possible.\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" + " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" + " lower buckets. Default: 0.95\n" + "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" + " Default: 1\n" + "\nResult:\n" + "{\n" + " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " }\n" + " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" + "}\n" + "\n" + "A negative feerate is returned if no answer can be given.\n" + "\nExample:\n" + + HelpExampleCli("estimaterawfee", "6 0.9 1") + ); + + RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM, UniValue::VNUM}, true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + int nBlocks = request.params[0].get_int(); + double threshold = 0.95; + if (!request.params[1].isNull()) + threshold = request.params[1].get_real(); + FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; + if (!request.params[2].isNull()) { + int horizonInt = request.params[2].get_int(); + if (horizonInt < 0 || horizonInt > 2) { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); + } else { + horizon = (FeeEstimateHorizon)horizonInt; + } + } + UniValue result(UniValue::VOBJ); + CFeeRate feeRate; + EstimationResult buckets; + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + + result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); + result.push_back(Pair("decay", buckets.decay)); + result.push_back(Pair("scale", (int)buckets.scale)); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("pass", passbucket)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("fail", failbucket)); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- @@ -902,7 +993,9 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, true, {"nblocks","address","maxtries"} }, #endif // ENABLE_MINER { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, - { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} }, + { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, + + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 4e98a61ee197..29da7022cc47 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -23,6 +23,7 @@ #include "wallet/wallet.h" #include "wallet/walletdb.h" #endif +#include "warnings.h" #include "masternode/masternode-sync.h" #include "spork.h" @@ -57,6 +58,7 @@ UniValue getinfo(const JSONRPCRequest& request) "\nDEPRECATED. Returns an object containing various state info.\n" "\nResult:\n" "{\n" + " \"deprecation-warning\": \"...\" (string) warning that the getinfo command is deprecated and will be removed in a future version\n" " \"version\": xxxxx, (numeric) the server version\n" " \"protocolversion\": xxxxx, (numeric) the protocol version\n" " \"walletversion\": xxxxx, (numeric) the wallet version\n" @@ -65,7 +67,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" " \"timeoffset\": xxxxx, (numeric) the time offset\n" " \"connections\": xxxxx, (numeric) the number of connections\n" - " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" + " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"testnet\": true|false, (boolean) if the server is using testnet or not\n" " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" @@ -73,7 +75,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee set in " + CURRENCY_UNIT + "/kB\n" " \"relayfee\": x.xxxx, (numeric) minimum relay fee for transactions in " + CURRENCY_UNIT + "/kB\n" - " \"errors\": \"...\" (string) any error messages\n" + " \"errors\": \"...\" (string) any error messages\n" "}\n" "\nExamples:\n" + HelpExampleCli("getinfo", "") @@ -92,6 +94,8 @@ UniValue getinfo(const JSONRPCRequest& request) GetProxy(NET_IPV4, proxy); UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("deprecation-warning", "WARNING: getinfo is deprecated and will be fully removed in a future version." + " Projects should transition to using getblockchaininfo, getnetworkinfo, and getwalletinfo.")); obj.push_back(Pair("version", CLIENT_VERSION)); obj.push_back(Pair("protocolversion", PROTOCOL_VERSION)); #ifdef ENABLE_WALLET diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index efbcea2c6297..5c18296e27fb 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -19,8 +19,7 @@ #include "util.h" #include "utilstrencodings.h" #include "version.h" - -#include +#include "warnings.h" #include diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index b02c4bacb147..b5261072b6b3 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include // for to_upper() #include diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 533915318864..a2d0b3f1947d 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1073,7 +1073,7 @@ class CTransactionSignatureSerializer { // Serialize the script if (nInput != nIn) // Blank out other inputs' signatures - ::Serialize(s, CScriptBase()); + ::Serialize(s, CScript()); else SerializeScriptCode(s); // Serialize the nSequence diff --git a/src/script/ismine.cpp b/src/script/ismine.cpp index fe9215c67da2..ce6033fa70e6 100644 --- a/src/script/ismine.cpp +++ b/src/script/ismine.cpp @@ -11,7 +11,6 @@ #include "script/standard.h" #include "script/sign.h" -#include typedef std::vector valtype; diff --git a/src/script/script.h b/src/script/script.h index 77eb896503bc..ad24cbf036e2 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -8,6 +8,7 @@ #include "crypto/common.h" #include "prevector.h" +#include "serialize.h" #include #include @@ -401,6 +402,13 @@ class CScript : public CScriptBase CScript(std::vector::const_iterator pbegin, std::vector::const_iterator pend) : CScriptBase(pbegin, pend) { } CScript(const unsigned char* pbegin, const unsigned char* pend) : CScriptBase(pbegin, pend) { } + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(static_cast(*this)); + } + CScript& operator+=(const CScript& b) { insert(end(), b.begin(), b.end()); @@ -649,15 +657,6 @@ class CScript : public CScriptBase CScriptBase::clear(); shrink_to_fit(); } - - template - void Serialize(Stream& s) const { - s << *(CScriptBase*)this; - } - template - void Unserialize(Stream& s) { - s >> *(CScriptBase*)this; - } }; class CReserveScript diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 08b88017cdae..a60bd1f0ee1e 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -12,7 +12,6 @@ #include "script/standard.h" #include "uint256.h" -#include typedef std::vector valtype; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 4046f12c3ab9..328b3948a8f2 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -10,7 +10,6 @@ #include "util.h" #include "utilstrencodings.h" -#include typedef std::vector valtype; diff --git a/src/sync.cpp b/src/sync.cpp index 785b4f07cfd4..98b9b63a99c1 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -9,7 +9,6 @@ #include -#include #include #ifdef DEBUG_LOCKCONTENTION diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 156c19315c65..48c42554303b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -24,8 +24,7 @@ BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(dbwrapper) { // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; + for (bool obfuscate : {false, true}) { fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'k'; @@ -45,8 +44,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper) BOOST_AUTO_TEST_CASE(dbwrapper_batch) { // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; + for (bool obfuscate : {false, true}) { fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); @@ -82,8 +80,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) BOOST_AUTO_TEST_CASE(dbwrapper_iterator) { // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; + for (bool obfuscate : {false, true}) { fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); @@ -95,7 +92,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_iterator) uint256 in2 = InsecureRand256(); BOOST_CHECK(dbw.Write(key2, in2)); - std::unique_ptr it(const_cast(&dbw)->NewIterator()); + std::unique_ptr it(const_cast(dbw).NewIterator()); // Be sure to seek past the obfuscation key (if it exists) it->Seek(key); @@ -210,13 +207,8 @@ BOOST_AUTO_TEST_CASE(iterator_ordering) BOOST_CHECK(dbw.Write(key, value)); } - std::unique_ptr it(const_cast(&dbw)->NewIterator()); - for (int c=0; c<2; ++c) { - int seek_start; - if (c == 0) - seek_start = 0x00; - else - seek_start = 0x80; + std::unique_ptr it(const_cast(dbw).NewIterator()); + for (int seek_start : {0x00, 0x80}) { it->Seek((uint8_t)seek_start); for (int x=seek_start; x<256; ++x) { uint8_t key; @@ -286,13 +278,8 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) } } - std::unique_ptr it(const_cast(&dbw)->NewIterator()); - for (int c=0; c<2; ++c) { - int seek_start; - if (c == 0) - seek_start = 0; - else - seek_start = 5; + std::unique_ptr it(const_cast(dbw).NewIterator()); + for (int seek_start : {0, 5}) { snprintf(buf, sizeof(buf), "%d", seek_start); StringContentsSerializer seek_key(buf); it->Seek(seek_key); diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 28bec6374a29..44109f216127 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -9,7 +9,6 @@ #include #include -#include #include BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index d1dc6ed8058b..44013b94e3b8 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -14,7 +14,6 @@ #include "test/test_dash.h" -#include #include typedef std::vector valtype; diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 1eea585ac3ad..b1b894e42c9d 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -49,8 +49,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int blocknum = 0; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs @@ -74,20 +74,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, ++blocknum); block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(feeEst.estimateSmartFee(1, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(3, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(4, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(8, &answerFound, mpool) == feeEst.estimateFee(8) && answerFound == 8); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee); } } @@ -104,13 +98,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - if (i > 1) { + if (i % 2 == 0) { //At scale 2, test logic is only correct for even targets BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); } - else { - BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) { + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -139,10 +134,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); } - int answerFound; for (int i = 1; i < 10;i++) { BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(feeEst.estimateSmartFee(i, &answerFound, mpool).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); } // Mine all those transactions @@ -155,16 +148,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265); + mpool.removeForBlock(block, 266); block.clear(); BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 400 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) { + while (blocknum < 665) { for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; @@ -180,7 +173,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) block.clear(); } BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); - for (int i = 2; i < 10; i++) { + for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } @@ -190,7 +183,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateFee(i).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 01308cbbb312..24524cf8452d 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -5,6 +5,7 @@ #include #include "prevector.h" +#include "reverse_iterator.h" #include "serialize.h" #include "streams.h" @@ -56,13 +57,13 @@ class prevector_tester { for (const T& v : pre_vector) { local_check(v == real_vector[pos++]); } - BOOST_REVERSE_FOREACH(const T& v, pre_vector) { + for (const T& v : reverse_iterate(pre_vector)) { local_check(v == real_vector[--pos]); } for (const T& v : const_pre_vector) { local_check(v == real_vector[pos++]); } - BOOST_REVERSE_FOREACH(const T& v, const_pre_vector) { + for (const T& v : reverse_iterate(const_pre_vector)) { local_check(v == real_vector[--pos]); } CDataStream ss1(SER_DISK, 0); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 18282dd6cc13..0485d715b9a7 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -23,7 +23,6 @@ #include #include -#include #include #include diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 3f457919926b..eea7be4b4db0 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -142,4 +142,39 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_test) BOOST_CHECK(vBlocksMain[r].GetAncestor(ret->nHeight) == ret); } } + +BOOST_AUTO_TEST_CASE(findearliestatleast_edge_test) +{ + std::list blocks; + for (unsigned int timeMax : {100, 100, 100, 200, 200, 200, 300, 300, 300}) { + CBlockIndex* prev = blocks.empty() ? nullptr : &blocks.back(); + blocks.emplace_back(); + blocks.back().nHeight = prev ? prev->nHeight + 1 : 0; + blocks.back().pprev = prev; + blocks.back().BuildSkip(); + blocks.back().nTimeMax = timeMax; + } + + CChain chain; + chain.SetTip(&blocks.back()); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250)->nHeight, 6); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300)->nHeight, 6); + BOOST_CHECK(!chain.FindEarliestAtLeast(350)); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1)->nHeight, 0); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits::min())->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits::min())->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits::max()) - 1)->nHeight, 0); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits::max())); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits::max())); + BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits::max()) + 1)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index a440feee6cef..ecc6941d4d34 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -7,6 +7,7 @@ #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "crypto/sha256.h" #include "fs.h" #include "key.h" #include "validation.h" @@ -38,6 +39,7 @@ extern void noui_connect(); BasicTestingSetup::BasicTestingSetup(const std::string& chainName) { + SHA256AutoDetect(); RandomInit(); ECC_Start(); BLSInit(); diff --git a/src/timedata.cpp b/src/timedata.cpp index 6549ecd53e12..a5c5bd44f3ae 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -15,7 +15,6 @@ #include "utilstrencodings.h" #include "warnings.h" -#include static CCriticalSection cs_nTimeOffset; static int64_t nTimeOffset = 0; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index b7c07f17f492..50e73d95a8d2 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include #include diff --git a/src/txdb.cpp b/src/txdb.cpp index 13fe5b9109d6..fbcf8fc0b6c2 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -129,7 +129,7 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { CCoinsViewCursor *CCoinsViewDB::Cursor() const { - CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(&db)->NewIterator(), GetBestBlock()); + CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(db).NewIterator(), GetBestBlock()); /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 87470ae40c59..55590b4cd081 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -12,6 +12,7 @@ #include "policy/policy.h" #include "policy/fees.h" #include "random.h" +#include "reverse_iterator.h" #include "streams.h" #include "timedata.h" #include "util.h" @@ -129,7 +130,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashes // This maximizes the benefit of the descendant cache and guarantees that // setMemPoolChildren will be updated, an assumption made in // UpdateForDescendants. - BOOST_REVERSE_FOREACH(const uint256 &hash, vHashesToUpdate) { + for (const uint256 &hash : reverse_iterate(vHashesToUpdate)) { // we cache the in-mempool children to avoid duplicate updates setEntries setChildren; // calculate children from mapNextTx @@ -692,7 +693,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} removeAddressIndex(hash); removeSpentIndex(hash); } diff --git a/src/util.h b/src/util.h index 0804facb0906..6639300b4534 100644 --- a/src/util.h +++ b/src/util.h @@ -269,7 +269,7 @@ class ArgsManager * Return string argument or default value * * @param strArg Argument to get (e.g. "-foo") - * @param default (e.g. "1") + * @param strDefault (e.g. "1") * @return command-line argument or default value */ std::string GetArg(const std::string& strArg, const std::string& strDefault); @@ -278,7 +278,7 @@ class ArgsManager * Return integer argument or default value * * @param strArg Argument to get (e.g. "-foo") - * @param default (e.g. 1) + * @param nDefault (e.g. 1) * @return command-line argument (0 if invalid number) or default value */ int64_t GetArg(const std::string& strArg, int64_t nDefault); @@ -287,7 +287,7 @@ class ArgsManager * Return boolean argument or default value * * @param strArg Argument to get (e.g. "-foo") - * @param default (true or false) + * @param fDefault (true or false) * @return command-line argument or default value */ bool GetBoolArg(const std::string& strArg, bool fDefault); diff --git a/src/validation.cpp b/src/validation.cpp index 0623a5c9e15f..5a478a04f94b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -24,6 +24,7 @@ #include "pow.h" #include "primitives/block.h" #include "primitives/transaction.h" +#include "reverse_iterator.h" #include "script/script.h" #include "script/sigcache.h" #include "script/standard.h" @@ -2594,7 +2595,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c nHeight = nTargetHeight; // Connect new blocks. - BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { + for (CBlockIndex *pindexConnect : reverse_iterate(vpindexToConnect)) { if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. @@ -4357,11 +4358,6 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(void) { - if (gArgs.GetBoolArg("-zapwallettxes", false)) { - LogPrintf("Skipping mempool.dat because of zapwallettxes\n"); - return true; - } - const CChainParams& chainparams = Params(); int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); diff --git a/src/validation.h b/src/validation.h index dceb2a60a31c..daca7342ca1b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -272,14 +272,6 @@ void UnloadBlockIndex(); void ThreadScriptCheck(); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(); -/** Format a string that describes several potential problems detected by the core. - * strFor can have three values: - * - "rpc": get critical warnings, which should put the client in safe mode if non-empty - * - "statusbar": get all warnings - * - "gui": get all warnings, translated (where possible) for GUI - * This function only returns the highest priority warning of the set selected by strFor. - */ -std::string GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ bool GetTransaction(const uint256 &hash, CTransactionRef &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); /** Find the best known block, and make it the tip of the block chain */ diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 83279e2e2438..c986e5767821 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -12,7 +12,6 @@ #include #include -#include int CCrypter::BytesToKeySHA512AES(const std::vector& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const { diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 5581f77c9eb9..d8da1b943d36 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -18,7 +18,6 @@ #include #endif -#include #include // diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6d0362c7ba74..dc6bc0430d57 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -27,7 +27,6 @@ #include -#include std::string static EncodeDumpTime(int64_t nTime) { return DateTimeStrFormat("%Y-%m-%dT%H:%M:%SZ", nTime); diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index bd5dad18ca59..31e2f6a69994 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -16,7 +16,7 @@ void RegisterWalletRPCCommands(CRPCTable &t); * @param[in] request JSONRPCRequest that wishes to access a wallet * @return NULL if no wallet should be used, or a pointer to the CWallet */ -CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest&); +CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest& request); std::string HelpRequiringPassphrase(CWallet *); void EnsureWalletIsUnlocked(CWallet *); diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index 927aa980c198..d597bd911dcc 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -8,7 +8,6 @@ #include -#include #include extern CWallet* pwalletMain; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 58b8b69d0e5b..d9ded216ea01 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5141,13 +5141,19 @@ bool CWallet::ParameterInteraction() } } - // -zapwallettx implies a rescan - if (gArgs.GetBoolArg("-zapwallettxes", false)) { + int zapwallettxes = gArgs.GetArg("-zapwallettxes", 0); + // -zapwallettxes implies dropping the mempool on startup + if (zapwallettxes != 0 && gArgs.SoftSetBoolArg("-persistmempool", false)) { + LogPrintf("%s: parameter interaction: -zapwallettxes=%s -> setting -persistmempool=0\n", __func__, zapwallettxes); + } + + // -zapwallettxes implies a rescan + if (zapwallettxes != 0) { if (is_multiwallet) { return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes")); } if (gArgs.SoftSetBoolArg("-rescan", true)) { - LogPrintf("%s: parameter interaction: -zapwallettxes= -> setting -rescan=1\n", __func__); + LogPrintf("%s: parameter interaction: -zapwallettxes=%s -> setting -rescan=1\n", __func__, zapwallettxes); } } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index cf8457c43beb..4a8375135e20 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -20,7 +20,6 @@ #include -#include #include // @@ -97,23 +96,23 @@ bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { - return WriteIC(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); + return WriteIC(std::make_pair(std::string("cscript"), hash), redeemScript, false); } bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { - if (!WriteIC(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) { + if (!WriteIC(std::make_pair(std::string("watchmeta"), dest), keyMeta)) { return false; } - return WriteIC(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); + return WriteIC(std::make_pair(std::string("watchs"), dest), '1'); } bool CWalletDB::EraseWatchOnly(const CScript &dest) { - if (!EraseIC(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) { + if (!EraseIC(std::make_pair(std::string("watchmeta"), dest))) { return false; } - return EraseIC(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); + return EraseIC(std::make_pair(std::string("watchs"), dest)); } bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) @@ -326,7 +325,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, { wss.nWatchKeys++; CScript script; - ssKey >> *(CScriptBase*)(&script); + ssKey >> script; char fYes; ssValue >> fYes; if (fYes == '1') @@ -443,7 +442,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, else if (strType == "watchmeta") { CScript script; - ssKey >> *(CScriptBase*)(&script); + ssKey >> script; keyID = CScriptID(script); } @@ -476,7 +475,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, uint160 hash; ssKey >> hash; CScript script; - ssValue >> *(CScriptBase*)(&script); + ssValue >> script; if (!pwallet->LoadCScript(script)) { strErr = "Error reading wallet database: LoadCScript failed"; diff --git a/src/warnings.h b/src/warnings.h index a7aa6574264c..fd0ca5394233 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -14,6 +14,13 @@ void SetfLargeWorkForkFound(bool flag); bool GetfLargeWorkForkFound(); void SetfLargeWorkInvalidChainFound(bool flag); bool GetfLargeWorkInvalidChainFound(); +/** Format a string that describes several potential problems detected by the core. + * strFor can have three values: + * - "rpc": get critical warnings, which should put the client in safe mode if non-empty + * - "statusbar": get all warnings + * - "gui": get all warnings, translated (where possible) for GUI + * This function only returns the highest priority warning of the set selected by strFor. + */ std::string GetWarnings(const std::string& strFor); static const bool DEFAULT_TESTSAFEMODE = false; diff --git a/test/README.md b/test/README.md index 074d98c80f19..52907ddb1608 100644 --- a/test/README.md +++ b/test/README.md @@ -155,6 +155,26 @@ import pdb; pdb.set_trace() anywhere in the test. You will then be able to inspect variables, as well as call methods that interact with the dashd nodes-under-test. +If further introspection of the dashd instances themselves becomes +necessary, this can be accomplished by first setting a pdb breakpoint +at an appropriate location, running the test to that point, then using +`gdb` to attach to the process and debug. + +For instance, to attach to `self.node[1]` during a run: + +```bash +2017-06-27 14:13:56.686000 TestFramework (INFO): Initializing test directory /tmp/user/1000/testo9vsdjo3 +``` + +use the directory path to get the pid from the pid file: + +```bash +cat /tmp/user/1000/testo9vsdjo3/node1/regtest/dashd.pid +gdb /home/example/dashd +``` + +Note: gdb attach step may require `sudo` + ### Util tests Util tests can be run locally by running `test/util/bitcoin-util-test.py`. diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index b9488f9b431a..1a8094020a09 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -453,8 +453,7 @@ def run_test(self): self.stop_node(2) self.stop_node(3) self.nodes[1].encryptwallet("test") - self.nodes.pop(1) - wait_node(1) + bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [['-usehd=0']] * self.num_nodes) # This test is not meant to test fee estimation and we'd like diff --git a/test/functional/smartfees.py b/test/functional/smartfees.py index e541e8ad54cb..baf9d94480d9 100755 --- a/test/functional/smartfees.py +++ b/test/functional/smartfees.py @@ -116,8 +116,8 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): for i,e in enumerate(all_estimates): # estimate is for i+1 if e >= 0: valid_estimate = True - # estimatesmartfee should return the same result - assert_equal(node.estimatesmartfee(i+1)["feerate"], e) + if i >= 13: # for n>=14 estimatesmartfee(n/2) should be at least as high as estimatefee(n) + assert(node.estimatesmartfee((i+1)//2)["feerate"] > float(e) - delta) else: invalid_estimates += 1 diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index f6c326ac8fd7..55b8b481e658 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -601,7 +601,10 @@ def satoshi_round(amount): # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. def create_confirmed_utxos(fee, node, count): - node.generate(int(0.5*count)+101) + to_generate = int(0.5*count)+101 + while to_generate > 0: + node.generate(min(25, to_generate)) + to_generate -= 25 utxos = node.listunspent() iterations = count - len(utxos) addr1 = node.getnewaddress() diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 855fdbd0c31b..963f229b4287 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -4,77 +4,73 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the zapwallettxes functionality. -- start three bitcoind nodes -- create four transactions on node 0 - two are confirmed and two are - unconfirmed. -- restart node 1 and verify that both the confirmed and the unconfirmed +- start two dashd nodes +- create two transactions on node 0 - one is confirmed and one is unconfirmed. +- restart node 0 and verify that both the confirmed and the unconfirmed transactions are still available. -- restart node 0 and verify that the confirmed transactions are still - available, but that the unconfirmed transaction has been zapped. +- restart node 0 with zapwallettxes and persistmempool, and verify that both + the confirmed and the unconfirmed transactions are still available. +- restart node 0 with just zapwallettxed and verify that the confirmed + transactions are still available, but that the unconfirmed transaction has + been zapped. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - +from test_framework.util import (assert_equal, + assert_raises_jsonrpc, + ) class ZapWalletTXesTest (BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 3 - - def setup_network(self): - super().setup_network() - connect_nodes_bi(self.nodes,0,2) + self.num_nodes = 2 - def run_test (self): + def run_test(self): self.log.info("Mining blocks...") self.nodes[0].generate(1) self.sync_all() - self.nodes[1].generate(101) - self.sync_all() - - assert_equal(self.nodes[0].getbalance(), 500) - - txid0 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - txid1 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + self.nodes[1].generate(100) self.sync_all() + + # This transaction will be confirmed + txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 10) + self.nodes[0].generate(1) self.sync_all() - - txid2 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - txid3 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - - tx0 = self.nodes[0].gettransaction(txid0) - assert_equal(tx0['txid'], txid0) #tx0 must be available (confirmed) - - tx1 = self.nodes[0].gettransaction(txid1) - assert_equal(tx1['txid'], txid1) #tx1 must be available (confirmed) - - tx2 = self.nodes[0].gettransaction(txid2) - assert_equal(tx2['txid'], txid2) #tx2 must be available (unconfirmed) - - tx3 = self.nodes[0].gettransaction(txid3) - assert_equal(tx3['txid'], txid3) #tx3 must be available (unconfirmed) - - #restart bitcoind + + # This transaction will not be confirmed + txid2 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 20) + + # Confirmed and unconfirmed transactions are now in the wallet. + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop-start node0. Both confirmed and unconfirmed transactions remain in the wallet. + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop node0 and restart with zapwallettxes and persistmempool. The unconfirmed + # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. self.stop_node(0) - self.nodes[0] = self.start_node(0,self.options.tmpdir) - - tx3 = self.nodes[0].gettransaction(txid3) - assert_equal(tx3['txid'], txid3) #tx must be available (unconfirmed) - + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-persistmempool=1", "-zapwallettxes=2"]) + + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop node0 and restart with zapwallettxes, but not persistmempool. + # The unconfirmed transaction is zapped and is no longer in the wallet. self.stop_node(0) - - #restart bitcoind with zapwallettxes - self.nodes[0] = self.start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) - - assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) - #there must be an exception because the unconfirmed wallettx0 must be gone by now + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-zapwallettxes=2"]) - tx0 = self.nodes[0].gettransaction(txid0) - assert_equal(tx0['txid'], txid0) #tx0 (confirmed) must still be available because it was confirmed + # tx1 is still be available because it was confirmed + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + # This will raise an exception because the unconfirmed transaction has been zapped + assert_raises_jsonrpc(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) if __name__ == '__main__': - ZapWalletTXesTest ().main () + ZapWalletTXesTest().main()