Fast Cache - Downgrade - reupgrade protection and other improvements

benchmarks/bench_test.go

@@ -59,6 +59,7 @@ func commitTree(b *testing.B, t *iavl.MutableTree) {
 
 // queries random keys against live state. Keys are almost certainly not in the tree.
 func runQueriesFast(b *testing.B, t *iavl.MutableTree, keyLen int) {
+	require.True(b, t.IsFastCacheEnabled())
 	for i := 0; i < b.N; i++ {
 		q := randBytes(keyLen)
 		t.Get(q)
@@ -67,6 +68,7 @@ func runQueriesFast(b *testing.B, t *iavl.MutableTree, keyLen int) {
 
 // queries keys that are known to be in state
 func runKnownQueriesFast(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+	require.True(b, t.IsFastCacheEnabled()) // to ensure fast storage is enabled
 	l := int32(len(keys))
 	for i := 0; i < b.N; i++ {
 		q := keys[rand.Int31n(l)]
@@ -75,22 +77,43 @@ func runKnownQueriesFast(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
 }
 
 func runQueriesSlow(b *testing.B, t *iavl.MutableTree, keyLen int) {
+	b.StopTimer()
+	// Save version to get an old immutable tree to query against,
+	// Fast storage is not enabled on old tree versions, allowing us to bench the desired behavior.
+	_, version, err := t.SaveVersion()
+	require.NoError(b, err)
+
+	itree, err := t.GetImmutable(version - 1)
+	require.NoError(b, err)
+	require.False(b, itree.IsFastCacheEnabled()) // to ensure fast storage is not enabled
+
+	b.StartTimer()
 	for i := 0; i < b.N; i++ {
 		q := randBytes(keyLen)
-		t.GetWithIndex(q)
+		itree.GetWithIndex(q)
 	}
 }
 
 func runKnownQueriesSlow(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+	b.StopTimer()
+	// Save version to get an old immutable tree to query against,
+	// Fast storage is not enabled on old tree versions, allowing us to bench the desired behavior.
+	_, version, err := t.SaveVersion()
+	require.NoError(b, err)
+
+	itree, err := t.GetImmutable(version - 1)
+	require.NoError(b, err)
+	require.False(b, itree.IsFastCacheEnabled()) // to ensure fast storage is not enabled
+	b.StartTimer()
 	l := int32(len(keys))
 	for i := 0; i < b.N; i++ {
 		q := keys[rand.Int31n(l)]
-		t.GetWithIndex(q)
+		itree.GetWithIndex(q)
 	}
 }
 
 func runIterationFast(b *testing.B, t *iavl.MutableTree, expectedSize int) {
-	require.True(b, t.IsFastCacheEnabled()) // to ensure that fast iterator is returned.
+	require.True(b, t.IsFastCacheEnabled()) // to ensure fast storage is enabled
 	for i := 0; i < b.N; i++ {
 		itr := t.ImmutableTree.Iterator(nil, nil, false)
 		iterate(b, itr, expectedSize)
@@ -100,7 +123,7 @@ func runIterationFast(b *testing.B, t *iavl.MutableTree, expectedSize int) {
 
 func runIterationSlow(b *testing.B, t *iavl.MutableTree, expectedSize int) {
 	for i := 0; i < b.N; i++ {
-		itr := iavl.NewIterator(nil, nil, false, t.ImmutableTree)
+		itr := iavl.NewIterator(nil, nil, false, t.ImmutableTree) // create slow iterator directly
 		iterate(b, itr, expectedSize)
 		itr.Close()
 	}

export_test.go

@@ -52,8 +52,8 @@ func setupExportTreeRandom(t *testing.T) *ImmutableTree {
 		keySize   = 16
 		valueSize = 16
 
-		versions    = 32   // number of versions to generate
-		versionOps  = 4096 // number of operations (create/update/delete) per version
+		versions    = 8   // number of versions to generate
+		versionOps  = 1024 // number of operations (create/update/delete) per version
 		updateRatio = 0.4  // ratio of updates out of all operations
 		deleteRatio = 0.2  // ratio of deletes out of all operations
 	)

fast_iterator.go

@@ -123,12 +123,11 @@ func (iter *FastIterator) Next() {
 
 // Close implements dbm.Iterator
 func (iter *FastIterator) Close() error {
-	iter.fastIterator = nil
-	iter.valid = false
-
 	if iter.fastIterator != nil {
 		iter.err = iter.fastIterator.Close()
 	}
+	iter.valid = false
+	iter.fastIterator = nil
 	return iter.err
 }
 

fast_node.go

@@ -12,7 +12,6 @@ type FastNode struct {
 	key                  []byte
 	versionLastUpdatedAt int64
 	value                []byte
-	// leafHash             []byte // TODO: Look into if this would help with proof stuff.
 }
 
 // NewFastNode returns a new fast node from a value and version.
@@ -34,7 +33,7 @@ func DeserializeFastNode(key []byte, buf []byte) (*FastNode, error) {
 
 	val, _, cause := decodeBytes(buf)
 	if cause != nil {
-		return nil, errors.Wrap(cause, "decoding node.value")
+		return nil, errors.Wrap(cause, "decoding fastnode.value")
 	}
 
 	fastNode := &FastNode{

fast_node_test.go

@@ -0,0 +1,58 @@
+package iavl
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFastNode_encodedSize(t *testing.T) {
+	fastNode := &FastNode{
+		key:                  randBytes(10),
+		versionLastUpdatedAt: 1,
+		value:                randBytes(20),
+	}
+
+	expectedSize := 1 + len(fastNode.value) + 1
+
+	require.Equal(t, expectedSize, fastNode.encodedSize())
+}
+
+func TestFastNode_encode_decode(t *testing.T) {
+	testcases := map[string]struct {
+		node        *FastNode
+		expectHex   string
+		expectError bool
+	}{
+		"nil":   {nil, "", true},
+		"empty": {&FastNode{}, "0000", false},
+		"inner": {&FastNode{
+			key:       []byte{0x4},
+			versionLastUpdatedAt:   1,
+			value: []byte{0x2},
+		}, "020102", false},
+	}
+	for name, tc := range testcases {
+		tc := tc
+		t.Run(name, func(t *testing.T) {
+			var buf bytes.Buffer
+			err := tc.node.writeBytes(&buf)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tc.expectHex, hex.EncodeToString(buf.Bytes()))
+
+			node, err := DeserializeFastNode(tc.node.key, buf.Bytes())
+			require.NoError(t, err)
+			// since value and leafHash are always decoded to []byte{} we augment the expected struct here
+			if tc.node.value == nil {
+				tc.node.value = []byte{}
+			}
+			require.Equal(t, tc.node, node)
+		})
+	}
+}

immutable_tree.go

@@ -148,7 +148,7 @@ func (t *ImmutableTree) Export() *Exporter {
 //
 // The index is the index in the list of leaf nodes sorted lexicographically by key. The leftmost leaf has index 0.
 // It's neighbor has index 1 and so on.
-func (t *ImmutableTree) GetWithIndex(key []byte) (index int64, value []byte) {
+func (t *ImmutableTree) GetWithIndex(key []byte) (int64, []byte) {
 	if t.root == nil {
 		return 0, nil
 	}
@@ -201,6 +201,7 @@ func (t *ImmutableTree) GetByIndex(index int64) (key []byte, value []byte) {
 	if t.root == nil {
 		return nil, nil
 	}
+
 	return t.root.getByIndex(t, index)
 }
 
@@ -212,7 +213,7 @@ func (t *ImmutableTree) Iterate(fn func(key []byte, value []byte) bool) bool {
 	}
 
 	itr := t.Iterator(nil, nil, true)
-
+	defer itr.Close()
 	for ; itr.Valid(); itr.Next() {
 		if fn(itr.Key(), itr.Value()) {
 			return true

mutable_tree.go

@@ -179,7 +179,7 @@ func (t *MutableTree) Iterate(fn func(key []byte, value []byte) bool) (stopped b
 	sort.Strings(unsavedFastNodesToSort)
 
 	itr := t.ImmutableTree.Iterator(nil, nil, true)
-
+	defer itr.Close()
 	nextUnsavedIdx := 0
 	for itr.Valid() && nextUnsavedIdx < len(unsavedFastNodesToSort) {
 		diskKeyStr := string(itr.Key())
@@ -439,11 +439,6 @@ func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) {
 	tree.mtx.Lock()
 	defer tree.mtx.Unlock()
 
-	// Attempt to upgrade
-	if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil {
-		return 0, err
-	}
-
 	tree.versions[targetVersion] = true
 
 	iTree := &ImmutableTree{
@@ -460,6 +455,11 @@ func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) {
 	tree.ImmutableTree = iTree
 	tree.lastSaved = iTree.clone()
 
+	// Attempt to upgrade
+	if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil {
+		return 0, err
+	}
+
 	return targetVersion, nil
 }
 
@@ -483,11 +483,6 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) {
 	tree.mtx.Lock()
 	defer tree.mtx.Unlock()
 
-	// Attempt to upgrade
-	if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil {
-		return 0, err
-	}
-
 	var latestRoot []byte
 	for version, r := range roots {
 		tree.versions[version] = true
@@ -524,6 +519,11 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) {
 	tree.lastSaved = t.clone()
 	tree.allRootLoaded = true
 
+	// Attempt to upgrade
+	if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil {
+		return 0, err
+	}
+
 	return latestVersion, nil
 }
 
@@ -588,6 +588,7 @@ func (tree *MutableTree) enableFastStorageAndCommit() error {
 	}()
 
 	itr := tree.ImmutableTree.Iterator(nil, nil, true)
+	defer itr.Close()
 	for ; itr.Valid(); itr.Next() {
 		if err = tree.ndb.SaveFastNode(NewFastNode(itr.Key(), itr.Value(), tree.version)); err != nil {
 			return err
@@ -671,7 +672,7 @@ func (tree *MutableTree) GetVersioned(key []byte, version int64) []byte {
 		if err != nil {
 			return nil
 		}
-		_, value := t.GetWithIndex(key)
+		value := t.Get(key)
 		return value
 	}
 	return nil
@@ -760,10 +761,8 @@ func (tree *MutableTree) saveFastNodeVersion() error {
 		return err
 	}
 
-	if !tree.ndb.isFastStorageEnabled() {
-		if err := tree.ndb.setStorageVersionBatch(fastStorageVersionValue); err != nil {
-			return err
-		}
+	if err := tree.ndb.setStorageVersionBatch(fastStorageVersionValue); err != nil {
+		return err
 	}
 
 	return nil

nodedb.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -18,6 +20,12 @@ const (
 	hashSize          = sha256.Size
 	genesisVersion    = 1
 	storageVersionKey = "storage_version"
+	// We store latest saved version together with storage version delimited by the constant below.
+	// This delimiter is valid only if fast storage is enabled (i.e. storageVersion >= fastStorageVersionValue).
+	// The latest saved version is needed for protection against downgrade and re-upgrade. In such a case, it would
+	// be possible to observe mismatch between the latest version state and the fast nodes on disk.
+	// Therefore, we would like to detect that and overwrite fast nodes on disk with the latest version state.
+	fastStorageVersionDelimiter = "-"
 	// Using semantic versioning: https://semver.org/
 	defaultStorageVersionValue = "1.0.0"
 	fastStorageVersionValue    = "1.1.0"
@@ -211,6 +219,10 @@ func (ndb *nodeDB) SaveFastNode(node *FastNode) error {
 }
 
 func (ndb *nodeDB) setStorageVersion(newVersion string) error {
+	if newVersion >= fastStorageVersionValue {
+		newVersion = newVersion + fastStorageVersionDelimiter + strconv.Itoa(int(ndb.latestVersion))
+	}
+
 	if err := ndb.db.Set(metadataKeyFormat.Key([]byte(storageVersionKey)), []byte(newVersion)); err != nil {
 		return err
 	}
@@ -219,6 +231,10 @@ func (ndb *nodeDB) setStorageVersion(newVersion string) error {
 }
 
 func (ndb *nodeDB) setStorageVersionBatch(newVersion string) error {
+	if newVersion >= fastStorageVersionValue {
+		newVersion = newVersion + fastStorageVersionDelimiter + strconv.Itoa(int(ndb.latestVersion))
+	}
+
 	if err := ndb.batch.Set(metadataKeyFormat.Key([]byte(storageVersionKey)), []byte(newVersion)); err != nil {
 		return err
 	}
@@ -234,6 +250,17 @@ func (ndb *nodeDB) isFastStorageEnabled() bool {
 	return ndb.getStorageVersion() >= fastStorageVersionValue
 }
 
+func (ndb *nodeDB) shouldForceFastStorageUpdate() bool {
+	versions := strings.Split(ndb.storageVersion, fastStorageVersionDelimiter)
+
+	if len(versions) == 2 {
+		if versions[1] != strconv.Itoa(int(ndb.getLatestVersion())) {
+			return true
+		}
+	}
+	return false
+}
+
 // SaveNode saves a FastNode to disk.
 func (ndb *nodeDB) saveFastNodeUnlocked(node *FastNode) error {
 	if node.key == nil {