test out the new iavl store that only stores in memory

Gopkg.toml

@@ -43,6 +43,7 @@ ignored = [
   "github.com/go-kit/kit*",
   "github.com/grpc-ecosystem/go-grpc-prometheus*",
   "github.com/prometheus/client_golang/prometheus*",
+  "github.com/loomnetwork/transfer-gateway*",
   "github.com/certusone/yubihsm-go*",
   "github.com/jmhodges/levigo*", # can only build it with the right c packages
   "github.com/btcsuite/btcd*"
@@ -59,8 +60,8 @@ ignored = [
 [[override]]
   name = "github.com/tendermint/tendermint"
   source = "https://github.com/loomnetwork/tendermint.git"
-  revision = "29bb7b1ad483bec3c797ce24a5135465fccfa5f6"
-
+  branch = "loomchain"
+  
 [[constraint]]
   name = "github.com/gomodule/redigo"
   version = "2.0.0"
@@ -76,8 +77,8 @@ ignored = [
 
 [[override]]
   name = "github.com/tendermint/iavl"
-  source = "https://github.com/enlight/iavl.git"
-  branch = "multi-mutex-nodedb"
+  source = "https://github.com/loomnetwork/iavl.git"
+  branch = "tmreal2"
 
 [[override]]
   name = "github.com/tendermint/go-amino"
@@ -94,7 +95,7 @@ ignored = [
 [[constraint]]
   name = "github.com/btcsuite/btcutil"
   revision = "9e5f4b9a998d263e3ce9c56664a7816001ac8000"
-
+  
 [prune]
   go-tests = true
   unused-packages = true

builtin/plugins/karma/test_helper.go

@@ -42,7 +42,7 @@ func MockStateWithKarmaAndCoinB(b *testing.B, karmaInit *ktypes.KarmaInitRequest
 }
 
 func MockStateWithKarmaAndCoin(karmaInit *ktypes.KarmaInitRequest, coinInit *ctypes.InitRequest, appDb db.DB) (loomchain.State, registry.Registry, vm.VM, error) {
-	appStore, err := store.NewIAVLStore(appDb, 0, 0)
+	appStore, err := store.NewIAVLStore(appDb, 0, 0, 0)
 	if err != nil {
 		return nil, nil, nil, err
 	}

cmd/loom/db/app.go

@@ -67,7 +67,7 @@ func newGetAppHeightCommand() *cobra.Command {
 			}
 			defer db.Close()
 
-			iavlStore, err := store.NewIAVLStore(db, 0, 0)
+			iavlStore, err := store.NewIAVLStore(db, 0, 0, 0)
 			if err != nil {
 				return err
 			}

cmd/loom/db/evm.go

@@ -44,7 +44,7 @@ func newDumpEVMStateCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			appStore, err := store.NewIAVLStore(db, 0, appHeight)
+			appStore, err := store.NewIAVLStore(db, 0, appHeight, 0)
 			if err != nil {
 				return err
 			}
@@ -176,7 +176,7 @@ func newDumpEVMStateMultiWriterAppStoreCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			iavlStore, err := store.NewIAVLStore(db, 0, appHeight)
+			iavlStore, err := store.NewIAVLStore(db, 0, appHeight, 0)
 			if err != nil {
 				return err
 			}

cmd/loom/loom.go

@@ -578,17 +578,18 @@ func loadAppStore(cfg *config.Config, logger *loom.Logger, targetVersion int64)
 		if cfg.AppStore.PruneInterval > int64(0) {
 			logger.Info("Loading Pruning IAVL Store")
 			appStore, err = store.NewPruningIAVLStore(db, store.PruningIAVLStoreConfig{
-				MaxVersions: cfg.AppStore.MaxVersions,
-				BatchSize:   cfg.AppStore.PruneBatchSize,
-				Interval:    time.Duration(cfg.AppStore.PruneInterval) * time.Second,
-				Logger:      logger,
+				MaxVersions:   cfg.AppStore.MaxVersions,
+				BatchSize:     cfg.AppStore.PruneBatchSize,
+				Interval:      time.Duration(cfg.AppStore.PruneInterval) * time.Second,
+				Logger:        logger,
+				FlushInterval: cfg.AppStore.IAVLFlushInterval,
 			})
 			if err != nil {
 				return nil, err
 			}
 		} else {
 			logger.Info("Loading IAVL Store")
-			appStore, err = store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion)
+			appStore, err = store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion, cfg.AppStore.IAVLFlushInterval)
 			if err != nil {
 				return nil, err
 			}
@@ -608,7 +609,7 @@ func loadAppStore(cfg *config.Config, logger *loom.Logger, targetVersion int64)
 		}
 	} else if cfg.AppStore.Version == 3 {
 		logger.Info("Loading Multi-Writer App Store")
-		iavlStore, err := store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion)
+		iavlStore, err := store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion, cfg.AppStore.IAVLFlushInterval)
 		if err != nil {
 			return nil, err
 		}

store/config.go

@@ -27,6 +27,8 @@ type AppStoreConfig struct {
 	// If true the app store will write EVM state to both IAVLStore and EvmStore
 	// This config works with AppStore Version 3 (MultiWriterAppStore) only
 	SaveEVMStateToIAVL bool
+
+	IAVLFlushInterval int64
 }
 
 func DefaultConfig() *AppStoreConfig {
@@ -42,6 +44,7 @@ func DefaultConfig() *AppStoreConfig {
 		NodeCacheSize:        10000,
 		SnapshotVersion:      MultiReaderIAVLStoreSnapshotV1,
 		SaveEVMStateToIAVL:   false,
+		IAVLFlushInterval:    0, //default to zero until we know its ready
 	}
 }
 

store/iavlstore.go

@@ -45,8 +45,9 @@ func init() {
 }
 
 type IAVLStore struct {
-	tree        *iavl.MutableTree
-	maxVersions int64 // maximum number of versions to keep when pruning
+	tree          *iavl.MutableTree
+	maxVersions   int64 // maximum number of versions to keep when pruning
+	flushInterval int64 // how often we persist to disk
 }
 
 func (s *IAVLStore) Delete(key []byte) {
@@ -135,7 +136,21 @@ func (s *IAVLStore) SaveVersion() ([]byte, int64, error) {
 	}(time.Now())
 
 	oldVersion := s.Version()
-	hash, version, err := s.tree.SaveVersion()
+
+	var version int64
+	var hash []byte
+	//Every X versions we should persist to disk
+	if s.flushInterval == 0 || ((oldVersion+1)%s.flushInterval == 0) {
+		if s.flushInterval != 0 {
+			log.Error(fmt.Sprintf("Flushing mem to disk at version %d\n", oldVersion+1))
+			hash, version, err = s.tree.FlushMemVersionDisk()
+		} else {
+			hash, version, err = s.tree.SaveVersion()
+		}
+	} else {
+		hash, version, err = s.tree.SaveVersionMem()
+	}
+
 	if err != nil {
 		return nil, 0, errors.Wrapf(err, "failed to save tree version %d", oldVersion+1)
 	}
@@ -180,7 +195,7 @@ func (s *IAVLStore) GetSnapshot() Snapshot {
 // old versions will never been deleted.
 // targetVersion can be used to load any previously saved version of the store, if set to zero then
 // the last version that was saved will be loaded.
-func NewIAVLStore(db dbm.DB, maxVersions, targetVersion int64) (*IAVLStore, error) {
+func NewIAVLStore(db dbm.DB, maxVersions, targetVersion, flushInterval int64) (*IAVLStore, error) {
 	tree := iavl.NewMutableTree(db, 10000)
 	_, err := tree.LoadVersion(targetVersion)
 	if err != nil {
@@ -193,8 +208,9 @@ func NewIAVLStore(db dbm.DB, maxVersions, targetVersion int64) (*IAVLStore, erro
 	}
 
 	return &IAVLStore{
-		tree:        tree,
-		maxVersions: maxVersions,
+		tree:          tree,
+		maxVersions:   maxVersions,
+		flushInterval: flushInterval,
 	}, nil
 }