Implement ldb database functionality for optional addr index.

db.go

@@ -9,16 +9,21 @@ import (
 
 	"github.com/btcsuite/btcutil"
 	"github.com/btcsuite/btcwire"
+	"golang.org/x/crypto/ripemd160"
 )
 
 // Errors that the various database functions may return.
 var (
+	ErrAddrIndexDoesNotExist  = errors.New("address index hasn't been built up yet")
+	ErrUnsupportedAddressType = errors.New("address type is not supported " +
+		"by the address-index")
 	ErrPrevShaMissing  = errors.New("previous sha missing from database")
 	ErrTxShaMissing    = errors.New("requested transaction does not exist")
 	ErrBlockShaMissing = errors.New("requested block does not exist")
 	ErrDuplicateSha    = errors.New("duplicate insert attempted")
 	ErrDbDoesNotExist  = errors.New("non-existent database")
 	ErrDbUnknownType   = errors.New("non-existent database type")
+	ErrNotImplemented  = errors.New("method has not yet been implemented")
 )
 
 // AllShas is a special value that can be used as the final sha when requesting
@@ -106,6 +111,34 @@ type Db interface {
 	// the database yet.
 	NewestSha() (sha *btcwire.ShaHash, height int64, err error)
 
+	// FetchAddrIndexTip returns the hash and block height of the most recent
+	// block which has had its address index populated. It will return
+	// ErrAddrIndexDoesNotExist along with a zero hash, and -1 if the
+	// addrindex hasn't yet been built up.
+	FetchAddrIndexTip() (sha *btcwire.ShaHash, height int64, err error)
+
+	// UpdateAddrIndexForBlock updates the stored addrindex with passed
+	// index information for a particular block height. Additionally, it
+	// will update the stored meta-data related to the curent tip of the
+	// addr index. These two operations are performed in an atomic
+	// transaction which is commited before the function returns.
+	// Addresses are indexed by the raw bytes of their base58 decoded
+	// hash160.
+	UpdateAddrIndexForBlock(blkSha *btcwire.ShaHash, height int64,
+		addrIndex BlockAddrIndex) error
+
+	// FetchTxsForAddr looks up and returns all transactions which either
+	// spend a previously created output of the passed address, or create
+	// a new output locked to the passed address. The, `limit` parameter
+	// should be the max number of transactions to be returned.
+	// Additionally, if the caller wishes to skip forward in the results
+	// some amount, the 'seek' represents how many results to skip.
+	// NOTE: Values for both `seek` and `limit` MUST be positive.
+	FetchTxsForAddr(addr btcutil.Address, skip int, limit int) ([]*TxListReply, error)
+
+	// DeleteAddrIndex deletes the entire addrindex stored within the DB.
+	DeleteAddrIndex() error
+
 	// RollbackClose discards the recent database changes to the previously
 	// saved data at last Sync and closes the database.
 	RollbackClose() (err error)
@@ -134,6 +167,14 @@ type TxListReply struct {
 	Err     error
 }
 
+// AddrIndexKeySize is the number of bytes used by keys into the BlockAddrIndex.
+const AddrIndexKeySize = ripemd160.Size
+
+// BlockAddrIndex represents the indexing structure for addresses.
+// It maps a hash160 to a list of transaction locations within a block that
+// either pays to or spends from the passed UTXO for the hash160.
+type BlockAddrIndex map[[AddrIndexKeySize]byte][]*btcwire.TxLoc
+
 // driverList holds all of the registered database backends.
 var driverList []DriverDB
 

db_test.go

@@ -19,7 +19,7 @@ var (
 	ignoreDbTypes = map[string]bool{"createopenfail": true}
 )
 
-// testNewestShaEmpty ensures the NewestSha returns the values expected by
+// testNewestShaEmpty ensures that NewestSha returns the values expected by
 // the interface contract.
 func testNewestShaEmpty(t *testing.T, db btcdb.Db) {
 	sha, height, err := db.NewestSha()

ldb/block.go

@@ -290,3 +290,41 @@ func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error)
 
 	return &sha, db.lastBlkIdx, nil
 }
+
+// fetchAddrIndexTip returns the last block height and block sha to be indexed.
+// Meta-data about the address tip is currently cached in memory, and will be
+// updated accordingly by functions that modify the state. This function is
+// used on start up to load the info into memory. Callers will use the public
+// version of this function below, which returns our cached copy.
+func (db *LevelDb) fetchAddrIndexTip() (*btcwire.ShaHash, int64, error) {
+	db.dbLock.Lock()
+	defer db.dbLock.Unlock()
+
+	data, err := db.lDb.Get(addrIndexMetaDataKey, db.ro)
+	if err != nil {
+		return &btcwire.ShaHash{}, -1, btcdb.ErrAddrIndexDoesNotExist
+	}
+
+	var blkSha btcwire.ShaHash
+	blkSha.SetBytes(data[0:32])
+
+	blkHeight := binary.LittleEndian.Uint64(data[32:])
+
+	return &blkSha, int64(blkHeight), nil
+}
+
+// FetchAddrIndexTip returns the hash and block height of the most recent
+// block whose transactions have been indexed by address. It will return
+// ErrAddrIndexDoesNotExist along with a zero hash, and -1 if the
+// addrindex hasn't yet been built up.
+func (db *LevelDb) FetchAddrIndexTip() (*btcwire.ShaHash, int64, error) {
+	db.dbLock.Lock()
+	defer db.dbLock.Unlock()
+
+	if db.lastAddrIndexBlkIdx == -1 {
+		return &btcwire.ShaHash{}, -1, btcdb.ErrAddrIndexDoesNotExist
+	}
+	sha := db.lastAddrIndexBlkSha
+
+	return &sha, db.lastAddrIndexBlkIdx, nil
+}

ldb/boundary_test.go

@@ -12,7 +12,8 @@ import (
 	"github.com/btcsuite/btcwire"
 )
 
-// we need to test for empty databas and make certain it returns proper value
+// we need to test for an empty database and make certain it returns the proper
+// values
 
 func TestEmptyDB(t *testing.T) {
 

ldb/dup_test.go

@@ -110,7 +110,7 @@ out:
 		lastSha = blkSha
 	}
 
-	// genrate a new block based on the last sha
+	// generate a new block based on the last sha
 	// these block are not verified, so there are a bunch of garbage fields
 	// in the 'generated' block.
 

ldb/insertremove_test.go

@@ -66,7 +66,7 @@ endtest:
 	for height := int64(0); height < int64(len(blocks)); height++ {
 
 		block := blocks[height]
-		// look up inputs to this x
+		// look up inputs to this tx
 		mblock := block.MsgBlock()
 		var txneededList []*btcwire.ShaHash
 		var txlookupList []*btcwire.ShaHash

ldb/internal_test.go

@@ -5,21 +5,60 @@
 package ldb
 
 import (
-	"fmt"
+	"bytes"
 
-	"github.com/btcsuite/btcdb"
-	"github.com/btcsuite/btcwire"
+	"testing"
+
+	"github.com/btcsuite/btcutil"
+	"golang.org/x/crypto/ripemd160"
 )
 
-// FetchSha returns the datablock and pver for the given ShaHash.
-// This is a testing only interface.
-func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32,
-	blkid int64, err error) {
-	sqldb, ok := db.(*LevelDb)
-	if !ok {
-		err = fmt.Errorf("invalid data type")
-		return
+func TestAddrIndexKeySerialization(t *testing.T) {
+	var hash160Bytes [ripemd160.Size]byte
+
+	fakeHash160 := btcutil.Hash160([]byte("testing"))
+	copy(fakeHash160, hash160Bytes[:])
+
+	fakeIndex := txAddrIndex{
+		hash160:   hash160Bytes,
+		blkHeight: 1,
+		txoffset:  5,
+		txlen:     360,
+	}
+
+	serializedKey := addrIndexToKey(&fakeIndex)
+	unpackedIndex := unpackTxIndex(serializedKey[22:])
+
+	if unpackedIndex.blkHeight != fakeIndex.blkHeight {
+		t.Errorf("Incorrect block height. Unpack addr index key"+
+			"serialization failed. Expected %d, received %d",
+			1, unpackedIndex.blkHeight)
+	}
+
+	if unpackedIndex.txoffset != fakeIndex.txoffset {
+		t.Errorf("Incorrect tx offset. Unpack addr index key"+
+			"serialization failed. Expected %d, received %d",
+			5, unpackedIndex.txoffset)
+	}
+
+	if unpackedIndex.txlen != fakeIndex.txlen {
+		t.Errorf("Incorrect tx len. Unpack addr index key"+
+			"serialization failed. Expected %d, received %d",
+			360, unpackedIndex.txlen)
+	}
+}
+
+func TestBytesPrefix(t *testing.T) {
+	testKey := []byte("a")
+
+	prefixRange := bytesPrefix(testKey)
+	if !bytes.Equal(prefixRange.Start, []byte("a")) {
+		t.Errorf("Wrong prefix start, got %d, expected %d", prefixRange.Start,
+			[]byte("a"))
+	}
+
+	if !bytes.Equal(prefixRange.Limit, []byte("b")) {
+		t.Errorf("Wrong prefix end, got %d, expected %d", prefixRange.Limit,
+			[]byte("b"))
 	}
-	buf, blkid, err = sqldb.fetchSha(sha)
-	return
 }

ldb/leveldb.go

@@ -53,6 +53,9 @@ type LevelDb struct {
 	lastBlkSha       btcwire.ShaHash
 	lastBlkIdx       int64
 
+	lastAddrIndexBlkSha btcwire.ShaHash
+	lastAddrIndexBlkIdx int64
+
 	txUpdateMap      map[btcwire.ShaHash]*txUpdateObj
 	txSpentUpdateMap map[btcwire.ShaHash]*spentTxUpdate
 }
@@ -92,7 +95,6 @@ func OpenDB(args ...interface{}) (btcdb.Db, error) {
 	}
 
 	// Need to find last block and tx
-
 	var lastknownblock, nextunknownblock, testblock int64
 
 	increment := int64(100000)
@@ -139,6 +141,14 @@ blocknarrow:
 		}
 	}
 
+	// Load the last block whose transactions have been indexed by address.
+	if sha, idx, err := ldb.fetchAddrIndexTip(); err == nil {
+		ldb.lastAddrIndexBlkSha = *sha
+		ldb.lastAddrIndexBlkIdx = idx
+	} else {
+		ldb.lastAddrIndexBlkIdx = -1
+	}
+
 	ldb.lastBlkSha = *lastSha
 	ldb.lastBlkIdx = lastknownblock
 	ldb.nextBlock = lastknownblock + 1
@@ -250,6 +260,7 @@ func CreateDB(args ...interface{}) (btcdb.Db, error) {
 	if err == nil {
 		ldb := db.(*LevelDb)
 		ldb.lastBlkIdx = -1
+		ldb.lastAddrIndexBlkIdx = -1
 		ldb.nextBlock = 0
 	}
 	return db, err