wallet: Traverse by height in calcNextBlake256Diff

This commit improves initial chain sync performance by switching the
calcNextBlake256Diff function to traverse the chain by height instead of
following the header hashes.

calcNextBlake256Diff is used to calculate the difficulty required for
block headers before DCP0011 is activated (which at this point in time
is the majority of the chain). This requires traversing the chain
backwards, down to the block prior to each work difficulty window change.

Previously this was done by following the header chain backwards. This
involved loading and decoding each header in sequence, in order to
extract the previous block hash, until each relevant block is reached.

This is a significant component of the cpu and memory loads for the
initial chain sync of the wallet.

This commit improves this process by switching the traversal of the
blockchain from being per block to being directly by height. Instead of
loading and decoding each header from the db, only the necessary headers
are accessed and only the required timestamp field is decoded instead of
the full header.

This is safe to do because, as the existing comment in the
ValidateHeaderChainDifficulties function explains, at the point of the
execution of calcNextBlake256Diff, the ancestor chain of the tested
header MUST be the wallet's current main chain. This implies that
accessing the block hashes by height will return the correct ancestor
nodes.

The net result (obtained via some ad-hoc profiling) is a reduction of
the overall sync time of about 50 seconds (spent entirely in processing)
and a decrease of about 15GB of allocated RAM (which further reduces GC
pressure).

wallet/difficulty.go

@@ -70,6 +70,9 @@ func (w *Wallet) findPrevTestNetDifficulty(dbtx walletdb.ReadTx, h *wire.BlockHe
 // calcNextBlake256Diff calculates the required difficulty for the block AFTER
 // the passed header based on the difficulty retarget rules for the blake256
 // hash algorithm used at Decred launch.
+//
+// The ancestor chain of the header being tested MUST be in the wallet's main
+// chain or in the passed chain slice.
 func (w *Wallet) calcNextBlake256Diff(dbtx walletdb.ReadTx, header *wire.BlockHeader,
 	chain []*BlockNode, newBlockTime time.Time) (uint32, error) {
 
@@ -119,29 +122,34 @@ func (w *Wallet) calcNextBlake256Diff(dbtx walletdb.ReadTx, header *wire.BlockHe
 
 	alpha := params.WorkDiffAlpha
 
-	// Number of nodes to traverse while calculating difficulty.
-	nodesToTraverse := (params.WorkDiffWindowSize * params.WorkDiffWindows)
+	// Number of windows to traverse while calculating difficulty.
+	windowsToTraverse := params.WorkDiffWindows
 
 	// Initialize bigInt slice for the percentage changes for each window period
 	// above or below the target.
 	windowChanges := make([]*big.Int, params.WorkDiffWindows)
 
 	// Regress through all of the previous blocks and store the percent changes
 	// per window period; use bigInts to emulate 64.32 bit fixed point.
+	//
+	// The regression is made by skipping to the block where each window change
+	// takes place (by height), therefore we assume that the header that is
+	// tested is a child of the wallet's main chain.
 	var olderTime, windowPeriod int64
 	var weights uint64
-	oldHeader := header
+	oldHeight := header.Height
 	recentTime := header.Timestamp.Unix()
 
+	ns := dbtx.ReadBucket(wtxmgrNamespaceKey)
+
 	for i := int64(0); ; i++ {
 		// Store and reset after reaching the end of every window period.
-		if i%params.WorkDiffWindowSize == 0 && i != 0 {
-			olderTime = oldHeader.Timestamp.Unix()
+		if i != 0 {
 			timeDifference := recentTime - olderTime
 
 			// Just assume we're at the target (no change) if we've
 			// gone all the way back to the genesis block.
-			if oldHeader.Height == 0 {
+			if oldHeight == 0 {
 				timeDifference = int64(params.TargetTimespan /
 					time.Second)
 			}
@@ -170,20 +178,29 @@ func (w *Wallet) calcNextBlake256Diff(dbtx walletdb.ReadTx, header *wire.BlockHe
 			recentTime = olderTime
 		}
 
-		if i == nodesToTraverse {
+		if i == windowsToTraverse {
 			break // Exit for loop when we hit the end.
 		}
 
 		// Get the previous node while staying at the genesis block as needed.
 		// Query the header from the provided chain instead of database if
 		// present.  The parent of chain[0] is guaranteed to be in stored in the
 		// database.
-		if oldHeader.Height != 0 {
-			if len(chain) > 0 && int32(oldHeader.Height)-int32(chain[0].Header.Height) > 0 {
-				oldHeader = chain[oldHeader.Height-chain[0].Header.Height-1].Header
+		if int64(oldHeight) > params.WorkDiffWindowSize {
+			oldHeight -= uint32(params.WorkDiffWindowSize)
+		} else {
+			oldHeight = 0
+		}
+		if oldHeight != 0 {
+			if len(chain) > 0 && int32(oldHeight)-int32(chain[0].Header.Height) >= 0 {
+				idx := oldHeight - chain[0].Header.Height
+				olderTime = chain[idx].Header.Timestamp.Unix()
 			} else {
-				var err error
-				oldHeader, err = w.txStore.GetBlockHeader(dbtx, &oldHeader.PrevBlock)
+				oldHeaderHash, err := w.txStore.GetMainChainBlockHashForHeight(ns, int32(oldHeight))
+				if err != nil {
+					return 0, err
+				}
+				olderTime, err = w.txStore.GetBlockHeaderTime(dbtx, &oldHeaderHash)
 				if err != nil {
 					return 0, err
 				}

wallet/udb/txmined.go

@@ -834,6 +834,17 @@ func (s *Store) GetSerializedBlockHeader(ns walletdb.ReadBucket, blockHash *chai
 	return fetchRawBlockHeader(ns, keyBlockHeader(blockHash))
 }
 
+// GetBlockHeaderTime returns the timestamp field of the header for the block
+// identified by its hash.
+func (s *Store) GetBlockHeaderTime(dbtx walletdb.ReadTx, blockHash *chainhash.Hash) (int64, error) {
+	ns := dbtx.ReadBucket(wtxmgrBucketKey)
+	v := ns.NestedReadBucket(bucketHeaders).Get(keyBlockHeader(blockHash))
+	if v == nil {
+		return 0, errors.E(errors.NotExist, "block header")
+	}
+	return int64(extractBlockHeaderUnixTime(v)), nil
+}
+
 // GetBlockHeader returns the block header for the block specified by its hash.
 func (s *Store) GetBlockHeader(dbtx walletdb.ReadTx, blockHash *chainhash.Hash) (*wire.BlockHeader, error) {
 	ns := dbtx.ReadBucket(wtxmgrBucketKey)