From 05fe1159ef505b9a5b893fab667923b672972882 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sat, 15 Jun 2019 15:07:20 +0200 Subject: [PATCH 01/32] test out the hack iavl store --- Gopkg.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 843d4bb6d1..483c6ea2fc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -77,8 +77,8 @@ ignored = [ [[override]] name = "github.com/tendermint/iavl" - source = "https://github.com/enlight/iavl.git" - branch = "multi-mutex-nodedb" + source = "https://github.com/loomnetwork/iavl.git" + branch = "hack1" [[override]] name = "github.com/tendermint/go-amino" From c85946cb5da80644fccec35763ba3b8b68f86289 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sat, 15 Jun 2019 15:11:30 +0200 Subject: [PATCH 02/32] update gopkg toml with hacked iavl store --- Gopkg.lock | 32 ++++++++++++++++++++------------ Gopkg.toml | 2 +- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2e7cf356a0..c5f321cb86 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -20,13 +20,9 @@ revision = "84a0ff3f153cbd7e280a19029a864bb04b504e62" [[projects]] - digest = "1:d17099fb1aae071174db067e43a02780bbe534a279e0506783acd33c4f4b131c" + digest = "1:95f4fdfbc8d7723bd21e467aabe98d86f06d3e2557b1ba92bf9df8aea6706784" name = "github.com/btcsuite/btcutil" - packages = [ - ".", - "base58", - "bech32", - ] + packages = ["base58"] pruneopts = "UT" revision = "9e5f4b9a998d263e3ce9c56664a7816001ac8000" @@ -54,6 +50,17 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" +[[projects]] + digest = "1:0594af97b2f4cec6554086eeace6597e20a4b69466eb4ada25adf9f4300dddd2" + name = "github.com/garyburd/redigo" + packages = [ + "internal", + "redis", + ] + pruneopts = "UT" + revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e" + version = "v1.6.0" + [[projects]] digest = "1:4062bc6de62d73e2be342243cf138cf499b34d558876db8d9430e2149388a4d8" name = "github.com/go-logfmt/logfmt" @@ -145,6 +152,7 @@ pruneopts = "UT" revision = "bd379c19d9638c72aea2c79b4ac0b973561e1783" + [[projects]] digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7" name = "github.com/magiconair/properties" @@ -301,13 +309,13 @@ version = "v0.14.0" [[projects]] - branch = "multi-mutex-nodedb" - digest = "1:c463c5a9951f564b1a7713406a7b0655d7964d234bca6d7d0fbb2789be55ce51" + branch = "hack1" + digest = "1:0d88ff8b3199707e101baccee83b1935171e3efa51cfda9e879787566be0585d" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "421b71b2dc0f0d7ad013a811383c293aa546c038" - source = "https://github.com/enlight/iavl.git" + revision = "8e3cfa5690fbd8e5c2b0f3ec9ac8ca34b860490b" + source = "https://github.com/loomnetwork/iavl.git" [[projects]] digest = "1:a1eb813569fbe2ba8c6b8e25c8b7a7acbc7f01d486362043d4494cf949308096" @@ -427,7 +435,7 @@ input-imports = [ "github.com/BurntSushi/toml", "github.com/allegro/bigcache", - "github.com/btcsuite/btcutil/base58", + "github.com/garyburd/redigo/redis", "github.com/golang/protobuf/proto", "github.com/gomodule/redigo/redis", "github.com/gorilla/websocket", @@ -445,11 +453,11 @@ "github.com/syndtr/goleveldb/leveldb/filter", "github.com/syndtr/goleveldb/leveldb/opt", "github.com/syndtr/goleveldb/leveldb/util", - "github.com/tendermint/btcd/btcec", "github.com/tendermint/go-amino", "github.com/tendermint/iavl", "github.com/tendermint/tendermint/abci/server", "github.com/tendermint/tendermint/abci/types", + "github.com/tendermint/tendermint/blockchain", "github.com/tendermint/tendermint/config", "github.com/tendermint/tendermint/crypto", "github.com/tendermint/tendermint/crypto/ed25519", diff --git a/Gopkg.toml b/Gopkg.toml index 483c6ea2fc..ee912b7032 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -95,7 +95,7 @@ ignored = [ [[constraint]] name = "github.com/btcsuite/btcutil" revision = "9e5f4b9a998d263e3ce9c56664a7816001ac8000" - + [prune] go-tests = true unused-packages = true From e528b08cfa03cf17116fb650ad7c12d9c5c70b4f Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sat, 15 Jun 2019 15:20:06 +0200 Subject: [PATCH 03/32] update to new iavl and make it compile --- Gopkg.lock | 5 ++--- store/multi_reader_iavl_store.go | 11 +---------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c5f321cb86..9615d1f328 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -152,7 +152,6 @@ pruneopts = "UT" revision = "bd379c19d9638c72aea2c79b4ac0b973561e1783" - [[projects]] digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7" name = "github.com/magiconair/properties" @@ -310,11 +309,11 @@ [[projects]] branch = "hack1" - digest = "1:0d88ff8b3199707e101baccee83b1935171e3efa51cfda9e879787566be0585d" + digest = "1:24c0dc01fbabb1b1ec8ea585f709712dc1cd7fd5750cc157445b83aa420a0fa5" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "8e3cfa5690fbd8e5c2b0f3ec9ac8ca34b860490b" + revision = "97ba0a732dbcb91b272ec62b0516f51c02194cd3" source = "https://github.com/loomnetwork/iavl.git" [[projects]] diff --git a/store/multi_reader_iavl_store.go b/store/multi_reader_iavl_store.go index 07bd401923..f988213449 100644 --- a/store/multi_reader_iavl_store.go +++ b/store/multi_reader_iavl_store.go @@ -171,16 +171,7 @@ func NewMultiReaderIAVLStore(nodeDB dbm.DB, valueDB db.DBWrapper, cfg *AppStoreC valueBatch: valueDB.NewBatch(), snapshotVersion: MultiReaderIAVLStoreSnapshotVersion(cfg.SnapshotVersion), } - var ndb iavl.NodeDB - switch cfg.NodeDBVersion { - case NodeDBV1: - ndb = iavl.NewNodeDB(nodeDB, cfg.NodeCacheSize, s.getValue) - case NodeDBV2: - ndb = iavl.NewNodeDB2(nodeDB, cfg.NodeCacheSize, s.getValue) - default: - return nil, errors.New("invalid AppStore.NodeDBVersion") - } - tree := iavl.NewMutableTreeWithNodeDB(ndb) + tree := iavl.NewMutableTree(nodeDB, 10) // load the latest saved tree treeVer, err := tree.LoadVersion(0) if err != nil { From 158b20508bd8dfa32342a30093ee1a55f2419b0f Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sat, 15 Jun 2019 18:35:36 +0200 Subject: [PATCH 04/32] update iavl hash --- Gopkg.lock | 11 ++--------- Gopkg.toml | 1 + 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 9615d1f328..eb98beb540 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -19,13 +19,6 @@ pruneopts = "UT" revision = "84a0ff3f153cbd7e280a19029a864bb04b504e62" -[[projects]] - digest = "1:95f4fdfbc8d7723bd21e467aabe98d86f06d3e2557b1ba92bf9df8aea6706784" - name = "github.com/btcsuite/btcutil" - packages = ["base58"] - pruneopts = "UT" - revision = "9e5f4b9a998d263e3ce9c56664a7816001ac8000" - [[projects]] branch = "master" digest = "1:78097abc20f73ec968b3f67bf74deda55009caa4b801d0d04f36145c869ab3c3" @@ -309,11 +302,11 @@ [[projects]] branch = "hack1" - digest = "1:24c0dc01fbabb1b1ec8ea585f709712dc1cd7fd5750cc157445b83aa420a0fa5" + digest = "1:57c585a9dcf1ad3666daa80a922b13beaa983274d2ca7c98fb86169b895426cf" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "97ba0a732dbcb91b272ec62b0516f51c02194cd3" + revision = "acd7b8c1e447e9c53e808c82da06fbe4fd5f63cf" source = "https://github.com/loomnetwork/iavl.git" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index ee912b7032..6dc6c3c99a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -44,6 +44,7 @@ ignored = [ "github.com/grpc-ecosystem/go-grpc-prometheus*", "github.com/prometheus/client_golang/prometheus*", "github.com/loomnetwork/yubihsm-go*", + "github.com/loomnetwork/transfer-gateway*", "github.com/certusone/yubihsm-go*", "github.com/jmhodges/levigo*", # can only build it with the right c packages "github.com/btcsuite/btcd*" From 7c1019de9c777a8500d9c95070269a099d622776 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 13:42:28 +0200 Subject: [PATCH 05/32] fix broken test --- store/store_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/store/store_test.go b/store/store_test.go index 0555cfe398..c43042def0 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -337,6 +337,7 @@ func (ts *StoreTestSuite) VerifyConcurrentSnapshots() { _, _, err := ts.store.SaveVersion() require.NoError(err) }() + wg.Wait() // readers for i := 0; i < 10; i++ { From 6bed507c92965a2f5e4badbcfd81bebe4779f18e Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 13:52:24 +0200 Subject: [PATCH 06/32] use the new saveversion mem function --- store/iavlstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/iavlstore.go b/store/iavlstore.go index 2a9e4309d4..44e70f0592 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -133,7 +133,7 @@ func (s *IAVLStore) SaveVersion() ([]byte, int64, error) { }(time.Now()) oldVersion := s.Version() - hash, version, err := s.tree.SaveVersion() + hash, version, err := s.tree.SaveVersionMem() if err != nil { return nil, 0, errors.Wrapf(err, "failed to save tree version %d", oldVersion+1) } From 14a78f0b5891aa036b01d204ac954f67cfe9ab54 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 14:00:29 +0200 Subject: [PATCH 07/32] add iavl persistence interval --- config/config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/config.go b/config/config.go index 561d10d637..3352aa4211 100755 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ import ( "path/filepath" "github.com/loomnetwork/loomchain/evm" + "github.com/tendermint/iavl" "github.com/loomnetwork/loomchain/auth" plasmacfg "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/config" @@ -135,6 +136,8 @@ type Config struct { EvmStore *evm.EvmStoreConfig + IAVLInterval int64 + // Dragons EVMDebugEnabled bool } @@ -378,6 +381,7 @@ func DefaultConfig() *Config { SessionDuration: 600, EVMAccountsEnabled: false, EVMDebugEnabled: false, + IAVLInterval: 100, Oracle: "", DeployEnabled: true, @@ -410,6 +414,8 @@ func DefaultConfig() *Config { cfg.FnConsensus = DefaultFnConsensusConfig() cfg.Auth = auth.DefaultConfig() + + iavl.MutateVersionInterval = cfg.IAVLInterval return cfg } From b0248bc45fc1e2c5a8c623bfa3091b8c7308aa0a Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 14:03:40 +0200 Subject: [PATCH 08/32] updating iavl test build --- Gopkg.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index eb98beb540..f1c2258e2d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -302,11 +302,11 @@ [[projects]] branch = "hack1" - digest = "1:57c585a9dcf1ad3666daa80a922b13beaa983274d2ca7c98fb86169b895426cf" + digest = "1:69bc0aaf2e705b5dbd6f4e943cba9d39f0464262d2b15294c006fbdc81e2d91f" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "acd7b8c1e447e9c53e808c82da06fbe4fd5f63cf" + revision = "b5f95efdee7df9ad037f5fb606d47899ece36a11" source = "https://github.com/loomnetwork/iavl.git" [[projects]] From 9661161cb37be38985c40ee2ffa4721ef98f4dee Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 14:41:12 +0200 Subject: [PATCH 09/32] bump iavl and tendermint --- Gopkg.lock | 9 +++++---- Gopkg.toml | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f1c2258e2d..acbadba1ba 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -302,15 +302,16 @@ [[projects]] branch = "hack1" - digest = "1:69bc0aaf2e705b5dbd6f4e943cba9d39f0464262d2b15294c006fbdc81e2d91f" + digest = "1:6913cddae31c79c4d931ce83c654f05593023b8328e3ca388b0ef4aef7cb5a4c" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "b5f95efdee7df9ad037f5fb606d47899ece36a11" + revision = "1bcb3af8d91ddc41042d6ed69cc0616a1662aac3" source = "https://github.com/loomnetwork/iavl.git" [[projects]] - digest = "1:a1eb813569fbe2ba8c6b8e25c8b7a7acbc7f01d486362043d4494cf949308096" + branch = "loomchain" + digest = "1:70d9461d1f2ba2ef6ecd33c03ebb5be662affdd2411785bcf17e33137a68ab1c" name = "github.com/tendermint/tendermint" packages = [ "abci/client", @@ -363,7 +364,7 @@ "version", ] pruneopts = "UT" - revision = "29bb7b1ad483bec3c797ce24a5135465fccfa5f6" + revision = "9d871e29f9d34ee6563a821c78e1a23b8984c822" source = "https://github.com/loomnetwork/tendermint.git" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index 6dc6c3c99a..bc1f16be15 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -61,8 +61,8 @@ ignored = [ [[override]] name = "github.com/tendermint/tendermint" source = "https://github.com/loomnetwork/tendermint.git" - revision = "29bb7b1ad483bec3c797ce24a5135465fccfa5f6" - + branch = "loomchain" + [[constraint]] name = "github.com/gomodule/redigo" version = "2.0.0" From c6e0c517983d718b05cea199022c87ffeaba94a4 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 14:58:52 +0200 Subject: [PATCH 10/32] refactored iavl flushing to be handled by the app --- builtin/plugins/karma/test_helper.go | 2 +- cmd/loom/db/app.go | 2 +- cmd/loom/db/evm.go | 4 ++-- cmd/loom/loom.go | 13 +++++++------ config/config.go | 6 ------ store/config.go | 3 +++ store/iavlstore.go | 28 ++++++++++++++++++++++------ store/pruning_iavlstore.go | 11 ++++++----- 8 files changed, 42 insertions(+), 27 deletions(-) diff --git a/builtin/plugins/karma/test_helper.go b/builtin/plugins/karma/test_helper.go index e4fae5e16c..d23c175dd6 100644 --- a/builtin/plugins/karma/test_helper.go +++ b/builtin/plugins/karma/test_helper.go @@ -42,7 +42,7 @@ func MockStateWithKarmaAndCoinB(b *testing.B, karmaInit *ktypes.KarmaInitRequest } func MockStateWithKarmaAndCoin(karmaInit *ktypes.KarmaInitRequest, coinInit *ctypes.InitRequest, appDb db.DB) (loomchain.State, registry.Registry, vm.VM, error) { - appStore, err := store.NewIAVLStore(appDb, 0, 0) + appStore, err := store.NewIAVLStore(appDb, 0, 0, 0) if err != nil { return nil, nil, nil, err } diff --git a/cmd/loom/db/app.go b/cmd/loom/db/app.go index 10162d5bbb..d0abb64c08 100644 --- a/cmd/loom/db/app.go +++ b/cmd/loom/db/app.go @@ -67,7 +67,7 @@ func newGetAppHeightCommand() *cobra.Command { } defer db.Close() - iavlStore, err := store.NewIAVLStore(db, 0, 0) + iavlStore, err := store.NewIAVLStore(db, 0, 0, 0) if err != nil { return err } diff --git a/cmd/loom/db/evm.go b/cmd/loom/db/evm.go index fbb5e890c9..91fc2eba29 100644 --- a/cmd/loom/db/evm.go +++ b/cmd/loom/db/evm.go @@ -44,7 +44,7 @@ func newDumpEVMStateCommand() *cobra.Command { if err != nil { return err } - appStore, err := store.NewIAVLStore(db, 0, appHeight) + appStore, err := store.NewIAVLStore(db, 0, appHeight, 0) if err != nil { return err } @@ -166,7 +166,7 @@ func newDumpEVMStateMultiWriterAppStoreCommand() *cobra.Command { if err != nil { return err } - iavlStore, err := store.NewIAVLStore(db, 0, appHeight) + iavlStore, err := store.NewIAVLStore(db, 0, appHeight, 0) if err != nil { return err } diff --git a/cmd/loom/loom.go b/cmd/loom/loom.go index 5caa6fb36e..21dd8a2db4 100644 --- a/cmd/loom/loom.go +++ b/cmd/loom/loom.go @@ -578,17 +578,18 @@ func loadAppStore(cfg *config.Config, logger *loom.Logger, targetVersion int64) if cfg.AppStore.PruneInterval > int64(0) { logger.Info("Loading Pruning IAVL Store") appStore, err = store.NewPruningIAVLStore(db, store.PruningIAVLStoreConfig{ - MaxVersions: cfg.AppStore.MaxVersions, - BatchSize: cfg.AppStore.PruneBatchSize, - Interval: time.Duration(cfg.AppStore.PruneInterval) * time.Second, - Logger: logger, + MaxVersions: cfg.AppStore.MaxVersions, + BatchSize: cfg.AppStore.PruneBatchSize, + Interval: time.Duration(cfg.AppStore.PruneInterval) * time.Second, + Logger: logger, + FlushInterval: cfg.AppStore.IAVLFlushInterval, }) if err != nil { return nil, err } } else { logger.Info("Loading IAVL Store") - appStore, err = store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion) + appStore, err = store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion, cfg.AppStore.IAVLFlushInterval) if err != nil { return nil, err } @@ -608,7 +609,7 @@ func loadAppStore(cfg *config.Config, logger *loom.Logger, targetVersion int64) } } else if cfg.AppStore.Version == 3 { logger.Info("Loading Multi-Writer App Store") - iavlStore, err := store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion) + iavlStore, err := store.NewIAVLStore(db, cfg.AppStore.MaxVersions, targetVersion, cfg.AppStore.IAVLFlushInterval) if err != nil { return nil, err } diff --git a/config/config.go b/config/config.go index 3352aa4211..561d10d637 100755 --- a/config/config.go +++ b/config/config.go @@ -11,7 +11,6 @@ import ( "path/filepath" "github.com/loomnetwork/loomchain/evm" - "github.com/tendermint/iavl" "github.com/loomnetwork/loomchain/auth" plasmacfg "github.com/loomnetwork/loomchain/builtin/plugins/plasma_cash/config" @@ -136,8 +135,6 @@ type Config struct { EvmStore *evm.EvmStoreConfig - IAVLInterval int64 - // Dragons EVMDebugEnabled bool } @@ -381,7 +378,6 @@ func DefaultConfig() *Config { SessionDuration: 600, EVMAccountsEnabled: false, EVMDebugEnabled: false, - IAVLInterval: 100, Oracle: "", DeployEnabled: true, @@ -414,8 +410,6 @@ func DefaultConfig() *Config { cfg.FnConsensus = DefaultFnConsensusConfig() cfg.Auth = auth.DefaultConfig() - - iavl.MutateVersionInterval = cfg.IAVLInterval return cfg } diff --git a/store/config.go b/store/config.go index bd43854f00..32006089fd 100644 --- a/store/config.go +++ b/store/config.go @@ -27,6 +27,8 @@ type AppStoreConfig struct { // If true the app store will write EVM state to both IAVLStore and EvmStore // This config works with AppStore Version 3 (MultiWriterAppStore) only SaveEVMStateToIAVL bool + + IAVLFlushInterval int64 } func DefaultConfig() *AppStoreConfig { @@ -42,6 +44,7 @@ func DefaultConfig() *AppStoreConfig { NodeCacheSize: 10000, SnapshotVersion: MultiReaderIAVLStoreSnapshotV1, SaveEVMStateToIAVL: false, + IAVLFlushInterval: 100, } } diff --git a/store/iavlstore.go b/store/iavlstore.go index 44e70f0592..9a6da54c15 100755 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -43,8 +43,9 @@ func init() { } type IAVLStore struct { - tree *iavl.MutableTree - maxVersions int64 // maximum number of versions to keep when pruning + tree *iavl.MutableTree + maxVersions int64 // maximum number of versions to keep when pruning + flushInterval int64 // how often we persist to disk } func (s *IAVLStore) Delete(key []byte) { @@ -133,7 +134,21 @@ func (s *IAVLStore) SaveVersion() ([]byte, int64, error) { }(time.Now()) oldVersion := s.Version() - hash, version, err := s.tree.SaveVersionMem() + + var version int64 + var hash []byte + //Every X versions we should persist to disk + if s.flushInterval == 0 || ((oldVersion+1)%s.flushInterval == 0) { + if s.flushInterval != 0 { + log.Error(fmt.Sprintf("Flushing mem to disk at version %d\n", oldVersion+1)) + hash, version, err = s.tree.FlushMemVersionDisk() + } else { + hash, version, err = s.tree.SaveVersion() + } + } else { + hash, version, err = s.tree.SaveVersionMem() + } + if err != nil { return nil, 0, errors.Wrapf(err, "failed to save tree version %d", oldVersion+1) } @@ -178,7 +193,7 @@ func (s *IAVLStore) GetSnapshot() Snapshot { // old versions will never been deleted. // targetVersion can be used to load any previously saved version of the store, if set to zero then // the last version that was saved will be loaded. -func NewIAVLStore(db dbm.DB, maxVersions, targetVersion int64) (*IAVLStore, error) { +func NewIAVLStore(db dbm.DB, maxVersions, targetVersion, flushInterval int64) (*IAVLStore, error) { tree := iavl.NewMutableTree(db, 10000) _, err := tree.LoadVersion(targetVersion) if err != nil { @@ -191,8 +206,9 @@ func NewIAVLStore(db dbm.DB, maxVersions, targetVersion int64) (*IAVLStore, erro } return &IAVLStore{ - tree: tree, - maxVersions: maxVersions, + tree: tree, + maxVersions: maxVersions, + flushInterval: flushInterval, }, nil } diff --git a/store/pruning_iavlstore.go b/store/pruning_iavlstore.go index 56e5cc6012..d654aa6f60 100644 --- a/store/pruning_iavlstore.go +++ b/store/pruning_iavlstore.go @@ -42,10 +42,11 @@ func init() { } type PruningIAVLStoreConfig struct { - MaxVersions int64 // maximum number of versions to keep when pruning - BatchSize int64 // maximum number of versions to delete in each cycle - Interval time.Duration - Logger *loom.Logger + MaxVersions int64 // maximum number of versions to keep when pruning + BatchSize int64 // maximum number of versions to delete in each cycle + FlushInterval int64 // number of versions before flushing to disk + Interval time.Duration + Logger *loom.Logger } // PruningIAVLStore is a specialized IAVLStore that has a background thread that periodically prunes @@ -72,7 +73,7 @@ func NewPruningIAVLStore(db dbm.DB, cfg PruningIAVLStoreConfig) (*PruningIAVLSto maxVersions = 2 } - store, err := NewIAVLStore(db, maxVersions, 0) + store, err := NewIAVLStore(db, maxVersions, 0, cfg.FlushInterval) if err != nil { return nil, err } From dd3311a71869d62388a65461305f58982bec36d6 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 15:03:02 +0200 Subject: [PATCH 11/32] fix test compilations --- store/multi_writer_app_store_test.go | 2 +- store/store_test.go | 8 ++++---- store/viewer/dbviewer.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/store/multi_writer_app_store_test.go b/store/multi_writer_app_store_test.go index 75156d1469..62f6960eb4 100644 --- a/store/multi_writer_app_store_test.go +++ b/store/multi_writer_app_store_test.go @@ -216,7 +216,7 @@ func (m *MultiWriterAppStoreTestSuite) TestMultiWriterAppStoreSaveVersion() { func mockMultiWriterStore() (*MultiWriterAppStore, error) { memDb, _ := db.LoadMemDB() - iavlStore, err := NewIAVLStore(memDb, 0, 0) + iavlStore, err := NewIAVLStore(memDb, 0, 0, 0) if err != nil { return nil, err } diff --git a/store/store_test.go b/store/store_test.go index c43042def0..ebff194d5a 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -386,7 +386,7 @@ func (ts *IAVLStoreTestSuite) SetupTest() { require := ts.Require() var err error db := dbm.NewMemDB() - ts.store, err = NewIAVLStore(db, 0, 0) + ts.store, err = NewIAVLStore(db, 0, 0, 0) require.NoError(err) } @@ -710,7 +710,7 @@ func TestPruningIAVLStoreKeepsAllVersionsIfMaxVersionsIsZero(t *testing.T) { } func TestIAVLStoreKeepsAllVersionsIfMaxVersionsIsZero(t *testing.T) { - store, err := NewIAVLStore(dbm.NewMemDB(), 0, 0) + store, err := NewIAVLStore(dbm.NewMemDB(), 0, 0, 0) require.NoError(t, err) require.Equal(t, int64(0), store.Version()) require.Equal(t, int64(0), store.maxVersions) @@ -744,7 +744,7 @@ func TestIAVLStoreKeepsAllVersionsIfMaxVersionsIsZero(t *testing.T) { func TestSwitchFromIAVLStoreToPruningIAVLStore(t *testing.T) { memDB := dbm.NewMemDB() - store1, err := NewIAVLStore(memDB, 0, 0) + store1, err := NewIAVLStore(memDB, 0, 0, 0) require.NoError(t, err) values := []struct { @@ -773,7 +773,7 @@ func TestSwitchFromIAVLStoreToPruningIAVLStore(t *testing.T) { require.Equal(t, int64(12), store1.Version()) - store2, err := NewIAVLStore(memDB, 11, 0) + store2, err := NewIAVLStore(memDB, 11, 0, 0) require.NoError(t, err) // force the store to prune an old version store2.Set(key1, val1) diff --git a/store/viewer/dbviewer.go b/store/viewer/dbviewer.go index b88e295b5b..2fcc284308 100644 --- a/store/viewer/dbviewer.go +++ b/store/viewer/dbviewer.go @@ -22,7 +22,7 @@ func investigate(name, path string) error { if err != nil { return err } - loomstore, err := store.NewIAVLStore(db, 0, 0) + loomstore, err := store.NewIAVLStore(db, 0, 0, 0) if err != nil { return err } From a9a1b204f9a7b9cfe2b64d8bdacb100160d3baf9 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 15:32:31 +0200 Subject: [PATCH 12/32] turn on flushing by default --- store/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/config.go b/store/config.go index 32006089fd..48dc888870 100644 --- a/store/config.go +++ b/store/config.go @@ -44,7 +44,7 @@ func DefaultConfig() *AppStoreConfig { NodeCacheSize: 10000, SnapshotVersion: MultiReaderIAVLStoreSnapshotV1, SaveEVMStateToIAVL: false, - IAVLFlushInterval: 100, + IAVLFlushInterval: 0, } } From 96c6ab18980cc1ea1646053fdd51b855ce077926 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 15:34:00 +0200 Subject: [PATCH 13/32] bump iavl version --- Gopkg.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index acbadba1ba..48bcf6affe 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -302,11 +302,11 @@ [[projects]] branch = "hack1" - digest = "1:6913cddae31c79c4d931ce83c654f05593023b8328e3ca388b0ef4aef7cb5a4c" + digest = "1:f769222f976983c2d24dcd693a9a7cf04b02bc6decd1721ee47aba0af3e99716" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "1bcb3af8d91ddc41042d6ed69cc0616a1662aac3" + revision = "6749242d3dbf563372ba475b0fe532eb3d9d7c68" source = "https://github.com/loomnetwork/iavl.git" [[projects]] From 33c7cee189d3a2f4086989a703cf5b1db7f2775b Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Sun, 16 Jun 2019 16:19:09 +0200 Subject: [PATCH 14/32] bump iavl version again --- Gopkg.lock | 4 ++-- store/config.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 48bcf6affe..679a4a296b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -302,11 +302,11 @@ [[projects]] branch = "hack1" - digest = "1:f769222f976983c2d24dcd693a9a7cf04b02bc6decd1721ee47aba0af3e99716" + digest = "1:fa0ec427da9a605bc7ca7af80e8547c883c0d316b11052f88095993151a8150b" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "6749242d3dbf563372ba475b0fe532eb3d9d7c68" + revision = "8f7fa9697dd37d4ca6537bca94878fdca4172f8d" source = "https://github.com/loomnetwork/iavl.git" [[projects]] diff --git a/store/config.go b/store/config.go index 48dc888870..32006089fd 100644 --- a/store/config.go +++ b/store/config.go @@ -44,7 +44,7 @@ func DefaultConfig() *AppStoreConfig { NodeCacheSize: 10000, SnapshotVersion: MultiReaderIAVLStoreSnapshotV1, SaveEVMStateToIAVL: false, - IAVLFlushInterval: 0, + IAVLFlushInterval: 100, } } From 5c07b171013106841dda766d4add21fcac5034db Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 18 Jun 2019 12:17:09 +0100 Subject: [PATCH 15/32] unit tests --- store/iavlstore_benchmark_test.go | 249 ++++++++++++++++++++++++++++++ store/iavlstore_test.go | 151 ++++++++++++++++++ 2 files changed, 400 insertions(+) create mode 100644 store/iavlstore_benchmark_test.go create mode 100644 store/iavlstore_test.go diff --git a/store/iavlstore_benchmark_test.go b/store/iavlstore_benchmark_test.go new file mode 100644 index 0000000000..27756f746e --- /dev/null +++ b/store/iavlstore_benchmark_test.go @@ -0,0 +1,249 @@ +package store + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" + "github.com/tendermint/iavl" + "github.com/tendermint/tendermint/libs/db" + + // "github.com/syndtr/goleveldb/leveldb" + + "github.com/loomnetwork/loomchain/log" +) + +const ( + checkDb = false +) + +type benchResult struct { + name string + timeS time.Duration + diskSize int64 + compactDiskSize int64 +} + +var ( + testno = int(0) + results map[string]benchResult +) + +type benchFunc func(b require.TestingT, name string) + +func TestBenchmark(t *testing.T) { + t.Skip() + log.Setup("debug", "file://-") + log.Root.With("module", "diff-iavlstore") + testno = 0 + os.RemoveAll("testdata") + _, err := os.Stat("testdata") + os.IsNotExist(err) + + results = make(map[string]benchResult) + + diskDbType = "goleveldb" + //diskDbType = "membd" + numBlocks = 5000 + blockSize = 100 + maxVersions = 2 + fmt.Println("numBlocks", numBlocks, "blockSize", blockSize) + blocks = nil + blocks = iavl.GenerateBlocks2(numBlocks, blockSize) + tree = iavl.NewMutableTree(db.NewMemDB(), 0) + for _, program := range blocks { + if err := program.Execute(tree); err != nil { + return + } + } + + t.Run("normal", benchNormal) + t.Run("flush10", benchFlush10) + t.Run("flush100", benchFlush100) + t.Run("MaxVers", benchMaxVersions) + + files, err := ioutil.ReadDir("./testdata") + for _, f := range files { + if !f.IsDir() { + continue + } + fName := f.Name() + if len(fName) <= 3 { + continue + } + fName = fName[:len(fName)-3] + size := DirSize(t, f) + + ldb, err := leveldb.OpenFile("./testdata/"+f.Name(), nil) + require.NoError(t, err) + require.NoError(t, ldb.CompactRange(util.Range{nil, nil})) + require.NoError(t, ldb.Close()) + + sizeCompact := DirSize(t, f) + if _, ok := results[fName]; ok { + r := results[fName] + r.diskSize = size + r.compactDiskSize = sizeCompact + results[fName] = r + } + + } + fmt.Println() + if _, ok := results["normal"]; ok { + nTime := results["normal"].timeS + nSize := results["normal"].diskSize + nCompactSize := results["normal"].compactDiskSize + + fmt.Println("name\ttime\tdisk size\tcompact\tratio time\tratio size\tratio compact") + for _, r := range results { + fmt.Printf( + "%s\t%v\t%v\t%v\t%v\t%v\t%v\n", + r.name, + r.timeS.Seconds(), + r.diskSize, + r.compactDiskSize, + uint64(r.timeS.Seconds()*100/nTime.Seconds()), + uint64(r.diskSize*100/nSize), + uint64(r.compactDiskSize*100/nCompactSize), + ) + } + } else { + fmt.Println("name\ttime\tdisk size\tcompact") + for _, r := range results { + fmt.Printf( + "%s\t%v\t%v\t%v\n", + r.name, + r.timeS.Seconds(), + r.diskSize, + r.compactDiskSize, + ) + } + } + +} +func benchNormal(t *testing.T) { + timeIavlStore(t, "normal", benchmarkNormal) +} +func benchFlush10(t *testing.T) { + timeIavlStore(t, "flush10", benchmarkFlush10) +} +func benchFlush50(t *testing.T) { + timeIavlStore(t, "flush50", benchmarkFlush50) +} +func benchFlush100(t *testing.T) { + timeIavlStore(t, "flush100", benchmarkFlush100) +} +func benchMaxVersions(t *testing.T) { + timeIavlStore(t, "maxVersions", benchmarkMaxVersions) +} + +func timeIavlStore(b require.TestingT, name string, fn benchFunc) { + startTime := time.Now() + + fn(b, name) + + now := time.Now() + elapsed := now.Sub(startTime) + results[name] = benchResult{ + name: name, + timeS: elapsed, + } + if checkDb { + diskDb, err := db.NewGoLevelDB(name, "./testdata") + diskTree := iavl.NewMutableTree(diskDb, 0) + _, err = diskTree.Load() + require.NoError(b, err) + diskTree.Iterate(func(key []byte, value []byte) bool { + _, treeValue := tree.Get(key) + require.Zero(b, bytes.Compare(value, treeValue)) + return false + }) + tree.Iterate(func(key []byte, value []byte) bool { + _, diskValue := diskTree.Get(key) + require.Zero(b, bytes.Compare(value, diskValue)) + return false + }) + diskDb.Close() + } +} + +func benchmarkNormal(b require.TestingT, name string) { + testno++ + diskDb := getDiskDb(b, name) + store, err := NewIAVLStore(diskDb, 0, 0, 0) + require.NoError(b, err) + executeBlocks(b, blocks, *store) + _, _, err = store.tree.SaveVersion() + require.NoError(b, err) + diskDb.Close() +} + +func benchmarkFlush10(b require.TestingT, name string) { + testno++ + diskDb := getDiskDb(b, name) + store, err := NewIAVLStore(diskDb, 0, 0, 10) + require.NoError(b, err) + executeBlocks(b, blocks, *store) + _, _, err = store.tree.SaveVersion() + require.NoError(b, err) + diskDb.Close() +} + +func benchmarkFlush50(b require.TestingT, name string) { + testno++ + diskDb := getDiskDb(b, name) + store, err := NewIAVLStore(diskDb, 0, 0, 50) + require.NoError(b, err) + executeBlocks(b, blocks, *store) + _, _, err = store.tree.SaveVersion() + require.NoError(b, err) + diskDb.Close() +} + +func benchmarkFlush100(b require.TestingT, name string) { + testno++ + diskDb := getDiskDb(b, name) + store, err := NewIAVLStore(diskDb, 0, 0, 100) + require.NoError(b, err) + executeBlocks(b, blocks, *store) + _, _, err = store.tree.SaveVersion() + require.NoError(b, err) + diskDb.Close() +} + +func benchmarkMaxVersions(b require.TestingT, name string) { + testno++ + diskDb := getDiskDb(b, name) + store, err := NewIAVLStore(diskDb, int64(maxVersions), 0, 0) + require.NoError(b, err) + executeBlocks(b, blocks, *store) + _, _, err = store.tree.SaveVersion() + require.NoError(b, err) + diskDb.Close() +} + +func DirSize(b require.TestingT, fi os.FileInfo) int64 { + var size int64 + cwd, err := os.Getwd() + require.NoError(b, err) + + path := filepath.Join(cwd, "testdata", fi.Name()) + err = filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return err + }) + require.NoError(b, err) + return size +} diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go new file mode 100644 index 0000000000..8b348643f0 --- /dev/null +++ b/store/iavlstore_test.go @@ -0,0 +1,151 @@ +package store + +import ( + "bytes" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tendermint/iavl" + "github.com/tendermint/tendermint/libs/db" + + "github.com/loomnetwork/loomchain/log" +) + +var ( + numBlocks = 10000 + blockSize = 1000 + + maxVersions = 2 + flushInterval = int64(4) + diskDbType = "memdb" + //diskDbType = "goleveldb" + blocks []*iavl.Program + tree *iavl.MutableTree +) + +func TestIavl(t *testing.T) { + numBlocks = 10 + blockSize = 10 + flushInterval = int64(9) + + log.Setup("debug", "file://-") + log.Root.With("module", "dual-iavlstore") + + os.RemoveAll("testdata") + _, err := os.Stat("testdata") + require.True(t, os.IsNotExist(err)) + + blocks = nil + blocks = iavl.GenerateBlocks2(numBlocks, blockSize) + tree = iavl.NewMutableTree(db.NewMemDB(), 0) + for _, program := range blocks { + require.NoError(t, program.Execute(tree)) + } + + t.Run("testFlush", testFlush) + t.Run("normal", testNormal) + t.Run("max versions", testMaxVersions) +} + +func testNormal(t *testing.T) { + diskDb := getDiskDb(t, "testNormal") + store, err := NewIAVLStore(diskDb, 0, 0, 0) + require.NoError(t, err) + executeBlocks(t, blocks, *store) + + diskTree := iavl.NewMutableTree(diskDb, 0) + _, err = diskTree.Load() + require.NoError(t, err) + for _, entry := range store.Range(nil) { + _, value := tree.Get(entry.Key) + require.Zero(t, bytes.Compare(value, entry.Value)) + _, diskValue := diskTree.Get(entry.Key) + require.Zero(t, bytes.Compare(value, diskValue)) + + } + fmt.Println() + tree.Iterate(func(key []byte, value []byte) bool { + require.Zero(t, bytes.Compare(value, store.Get(key))) + _, diskValue := diskTree.Get(key) + require.Zero(t, bytes.Compare(value, diskValue)) + return false + }) + diskDb.Close() +} + +func testFlush(t *testing.T) { + diskDb := getDiskDb(t, "testFlush") + store, err := NewIAVLStore(diskDb, 0, 0, flushInterval) + require.NoError(t, err) + executeBlocks(t, blocks, *store) + _, _, err = store.tree.SaveVersion() + + diskTree := iavl.NewMutableTree(diskDb, 0) + _, err = diskTree.Load() + require.NoError(t, err) + for _, entry := range store.Range(nil) { + _, value := tree.Get(entry.Key) + require.Zero(t, bytes.Compare(value, entry.Value)) + _, diskValue := diskTree.Get(entry.Key) + require.Zero(t, bytes.Compare(value, diskValue)) + } + tree.Iterate(func(key []byte, value []byte) bool { + require.Zero(t, bytes.Compare(value, store.Get(key))) + _, diskValue := diskTree.Get(key) + require.Zero(t, bytes.Compare(value, diskValue)) + return false + }) + diskDb.Close() +} + +func testMaxVersions(t *testing.T) { + diskDb := getDiskDb(t, "testMaxVersions") + store, err := NewIAVLStore(diskDb, int64(maxVersions), 0, 0) + require.NoError(t, err) + executeBlocks(t, blocks, *store) + + for i := 1; i <= numBlocks; i++ { + require.Equal(t, + i > numBlocks-maxVersions, + store.tree.VersionExists(int64(i)), + ) + } + diskTree := iavl.NewMutableTree(diskDb, 0) + _, err = diskTree.Load() + require.NoError(t, err) + for _, entry := range store.Range(nil) { + _, value := tree.Get(entry.Key) + require.Zero(t, bytes.Compare(value, entry.Value)) + _, diskValue := diskTree.Get(entry.Key) + require.Zero(t, bytes.Compare(value, diskValue)) + } + tree.Iterate(func(key []byte, value []byte) bool { + require.Zero(t, bytes.Compare(value, store.Get(key))) + _, diskValue := diskTree.Get(key) + require.Zero(t, bytes.Compare(value, diskValue)) + return false + }) + diskDb.Close() +} + +func executeBlocks(t require.TestingT, blocks []*iavl.Program, store IAVLStore) { + for _, block := range blocks { + require.NoError(t, block.Execute(store.tree)) + _, _, err := store.SaveVersion() + require.NoError(t, err) + require.NoError(t, store.Prune()) + } +} + +func getDiskDb(t require.TestingT, name string) db.DB { + if diskDbType == "goleveldb" { + diskDb, err := db.NewGoLevelDB(name, "./testdata") + require.NoError(t, err) + return diskDb + + } else { + return db.NewMemDB() + } +} From 0dabdee6f32db891720cb2cdbe246426b7bf7671 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Tue, 18 Jun 2019 12:38:21 +0100 Subject: [PATCH 16/32] touch --- store/iavlstore_benchmark_test.go | 2 +- store/iavlstore_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/store/iavlstore_benchmark_test.go b/store/iavlstore_benchmark_test.go index 27756f746e..d59997bcf7 100644 --- a/store/iavlstore_benchmark_test.go +++ b/store/iavlstore_benchmark_test.go @@ -43,7 +43,7 @@ func TestBenchmark(t *testing.T) { log.Setup("debug", "file://-") log.Root.With("module", "diff-iavlstore") testno = 0 - os.RemoveAll("testdata") + require.NoError(t, os.RemoveAll("testdata")) _, err := os.Stat("testdata") os.IsNotExist(err) diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index 8b348643f0..f4723d98c2 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -33,7 +33,7 @@ func TestIavl(t *testing.T) { log.Setup("debug", "file://-") log.Root.With("module", "dual-iavlstore") - os.RemoveAll("testdata") + require.NoError(t, os.RemoveAll("testdata")) _, err := os.Stat("testdata") require.True(t, os.IsNotExist(err)) From 1a029d83f5cbecf15f75b0c593047fcf8a46441b Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 19 Jun 2019 09:56:20 +0100 Subject: [PATCH 17/32] go-loom revision --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bdfab4eaf2..94718c3ad3 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ TRANSFER_GATEWAY_DIR=$(GOPATH)/src/$(PKG_TRANSFER_GATEWAY) # NOTE: To build on Jenkins using a custom go-loom branch update the `deps` target below to checkout # that branch, you only need to update GO_LOOM_GIT_REV if you wish to lock the build to a # specific commit. -GO_LOOM_GIT_REV = HEAD +GO_LOOM_GIT_REV = 3c6078c4fa4e24e9cc0100204c927c8c674900c8 # Specifies the loomnetwork/transfer-gateway branch/revision to use. TG_GIT_REV = HEAD # loomnetwork/go-ethereum loomchain branch From 9ac3c41dfc8db7bac7f08c004796c371b43797c4 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 21 Jun 2019 12:52:28 +0100 Subject: [PATCH 18/32] unit test --- store/iavlstore_test.go | 80 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index f4723d98c2..3367ac8823 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -25,9 +25,78 @@ var ( tree *iavl.MutableTree ) +func TestOrphans(t *testing.T) { + log.Setup("debug", "file://-") + log.Root.With("module", "dual-iavlstore") + + flushInterval = 5 + memDB := db.NewMemDB() + baseStore, err := NewIAVLStore(memDB, 0, 0, flushInterval) + require.NoError(t, err) + testOrphans(t, baseStore, memDB, flushInterval) + + memDB2 := db.NewMemDB() + flushStore, err := NewIAVLStore(memDB2, 0, 0, flushInterval) + require.NoError(t, err) + testOrphans(t, flushStore, memDB, flushInterval) +} + +func testOrphans(t *testing.T, store *IAVLStore, diskDb db.DB, flushInterval int64) { + store.Set([]byte("k1"), []byte("Fred")) + store.Set([]byte("k2"), []byte("John")) + for i := 0; i < int(flushInterval-1); i++ { + _, _, err := store.SaveVersion() + require.NoError(t, err) + } + store.Set([]byte("k2"), []byte("Bob")) + + _, _, err := store.SaveVersion() + + require.NoError(t, err) + + store.Set([]byte("k1"), []byte("Mary")) + store.Set([]byte("k2"), []byte("Sally")) + for i := 0; i < int(flushInterval)-1; i++ { + _, _, err := store.SaveVersion() + require.NoError(t, err) + } + + store.Set([]byte("k2"), []byte("Jim")) + for i := 0; i < 2*int(flushInterval); i++ { + _, _, err := store.SaveVersion() + require.NoError(t, err) + } + lastVersion := 3 * flushInterval + store = nil + + newStore, err := NewIAVLStore(diskDb, 0, int64(lastVersion), 0) + require.NoError(t, err) + + k1Value, _, err := newStore.tree.GetVersionedWithProof([]byte("k1"), flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte("Fred"), k1Value)) + + k1Value, _, err = newStore.tree.GetVersionedWithProof([]byte("k1"), 2*flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte("Mary"), k1Value)) + + k2Value, _, err := newStore.tree.GetVersionedWithProof([]byte("k2"), flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte("Bob"), k2Value)) + + k2Value, _, err = newStore.tree.GetVersionedWithProof([]byte("k2"), 2*flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte("Jim"), k2Value)) + + k2Value, _, err = newStore.tree.GetVersionedWithProof([]byte("k2"), 3*flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte("Jim"), k2Value)) +} + func TestIavl(t *testing.T) { - numBlocks = 10 - blockSize = 10 + t.Skip() + numBlocks = 1000 + blockSize = 100 flushInterval = int64(9) log.Setup("debug", "file://-") @@ -58,6 +127,8 @@ func testNormal(t *testing.T) { diskTree := iavl.NewMutableTree(diskDb, 0) _, err = diskTree.Load() require.NoError(t, err) + fmt.Println("testNormal") + //diskTree.PrintDiskDb() for _, entry := range store.Range(nil) { _, value := tree.Get(entry.Key) require.Zero(t, bytes.Compare(value, entry.Value)) @@ -85,6 +156,8 @@ func testFlush(t *testing.T) { diskTree := iavl.NewMutableTree(diskDb, 0) _, err = diskTree.Load() require.NoError(t, err) + fmt.Println("testFlush") + //diskTree.PrintDiskDb() for _, entry := range store.Range(nil) { _, value := tree.Get(entry.Key) require.Zero(t, bytes.Compare(value, entry.Value)) @@ -115,6 +188,9 @@ func testMaxVersions(t *testing.T) { diskTree := iavl.NewMutableTree(diskDb, 0) _, err = diskTree.Load() require.NoError(t, err) + + fmt.Println("testMaxVersions") + //diskTree.PrintDiskDb() for _, entry := range store.Range(nil) { _, value := tree.Get(entry.Key) require.Zero(t, bytes.Compare(value, entry.Value)) From f38a8f57f1ac343035b655bff41527e3ddbc62bf Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 21 Jun 2019 13:04:56 +0100 Subject: [PATCH 19/32] unit test --- store/iavlstore_test.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index 3367ac8823..1ddf6a872e 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -49,13 +49,14 @@ func testOrphans(t *testing.T, store *IAVLStore, diskDb db.DB, flushInterval int require.NoError(t, err) } store.Set([]byte("k2"), []byte("Bob")) - - _, _, err := store.SaveVersion() + store.Set([]byte("k3"), []byte("Harry")) + _, _, err := store.SaveVersion() // save to disk require.NoError(t, err) store.Set([]byte("k1"), []byte("Mary")) store.Set([]byte("k2"), []byte("Sally")) + store.Delete([]byte("k3")) for i := 0; i < int(flushInterval)-1; i++ { _, _, err := store.SaveVersion() require.NoError(t, err) @@ -63,7 +64,7 @@ func testOrphans(t *testing.T, store *IAVLStore, diskDb db.DB, flushInterval int store.Set([]byte("k2"), []byte("Jim")) for i := 0; i < 2*int(flushInterval); i++ { - _, _, err := store.SaveVersion() + _, _, err := store.SaveVersion() // save to disk require.NoError(t, err) } lastVersion := 3 * flushInterval @@ -91,12 +92,19 @@ func testOrphans(t *testing.T, store *IAVLStore, diskDb db.DB, flushInterval int k2Value, _, err = newStore.tree.GetVersionedWithProof([]byte("k2"), 3*flushInterval) require.NoError(t, err) require.Equal(t, 0, bytes.Compare([]byte("Jim"), k2Value)) + + k2Value, _, err = newStore.tree.GetVersionedWithProof([]byte("k3"), flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte("Harry"), k2Value)) + + k2Value, _, err = newStore.tree.GetVersionedWithProof([]byte("k3"), 2*flushInterval) + require.NoError(t, err) + require.Equal(t, 0, bytes.Compare([]byte(""), k2Value)) } func TestIavl(t *testing.T) { - t.Skip() - numBlocks = 1000 - blockSize = 100 + numBlocks = 10 + blockSize = 10 flushInterval = int64(9) log.Setup("debug", "file://-") From cb58e1c64ff3a6dd955f529edb304c66332f209b Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 21 Jun 2019 13:51:20 +0100 Subject: [PATCH 20/32] point to new iavl commit --- Gopkg.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index 679a4a296b..6572f8e83b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -306,7 +306,7 @@ name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "8f7fa9697dd37d4ca6537bca94878fdca4172f8d" + revision = "ea354ab86e708d9909f6e59099f3a2b2a7dceda4" source = "https://github.com/loomnetwork/iavl.git" [[projects]] From 846ff2d87e98badd01704eb72c803c331b04353e Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 21 Jun 2019 15:13:57 +0100 Subject: [PATCH 21/32] unit test --- store/iavlstore_benchmark_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/store/iavlstore_benchmark_test.go b/store/iavlstore_benchmark_test.go index d59997bcf7..e01996c2c3 100644 --- a/store/iavlstore_benchmark_test.go +++ b/store/iavlstore_benchmark_test.go @@ -39,7 +39,7 @@ var ( type benchFunc func(b require.TestingT, name string) func TestBenchmark(t *testing.T) { - t.Skip() + //t.Skip() log.Setup("debug", "file://-") log.Root.With("module", "diff-iavlstore") testno = 0 @@ -51,7 +51,7 @@ func TestBenchmark(t *testing.T) { diskDbType = "goleveldb" //diskDbType = "membd" - numBlocks = 5000 + numBlocks = 1000 blockSize = 100 maxVersions = 2 fmt.Println("numBlocks", numBlocks, "blockSize", blockSize) @@ -65,8 +65,8 @@ func TestBenchmark(t *testing.T) { } t.Run("normal", benchNormal) - t.Run("flush10", benchFlush10) - t.Run("flush100", benchFlush100) + t.Run("flush20", benchflush20) + //t.Run("flush100", benchflush100) t.Run("MaxVers", benchMaxVersions) files, err := ioutil.ReadDir("./testdata") @@ -131,14 +131,14 @@ func TestBenchmark(t *testing.T) { func benchNormal(t *testing.T) { timeIavlStore(t, "normal", benchmarkNormal) } -func benchFlush10(t *testing.T) { - timeIavlStore(t, "flush10", benchmarkFlush10) +func benchflush20(t *testing.T) { + timeIavlStore(t, "flush20", benchmarkflush20) } func benchFlush50(t *testing.T) { timeIavlStore(t, "flush50", benchmarkFlush50) } -func benchFlush100(t *testing.T) { - timeIavlStore(t, "flush100", benchmarkFlush100) +func benchflush100(t *testing.T) { + timeIavlStore(t, "flush100", benchmarkflush100) } func benchMaxVersions(t *testing.T) { timeIavlStore(t, "maxVersions", benchmarkMaxVersions) @@ -185,10 +185,10 @@ func benchmarkNormal(b require.TestingT, name string) { diskDb.Close() } -func benchmarkFlush10(b require.TestingT, name string) { +func benchmarkflush20(b require.TestingT, name string) { testno++ diskDb := getDiskDb(b, name) - store, err := NewIAVLStore(diskDb, 0, 0, 10) + store, err := NewIAVLStore(diskDb, 0, 0, 20) require.NoError(b, err) executeBlocks(b, blocks, *store) _, _, err = store.tree.SaveVersion() @@ -207,7 +207,7 @@ func benchmarkFlush50(b require.TestingT, name string) { diskDb.Close() } -func benchmarkFlush100(b require.TestingT, name string) { +func benchmarkflush100(b require.TestingT, name string) { testno++ diskDb := getDiskDb(b, name) store, err := NewIAVLStore(diskDb, 0, 0, 100) From f9227683a5640bec2ca8a9d5aa2fab5d1020bcec Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 21 Jun 2019 15:33:29 +0100 Subject: [PATCH 22/32] skip benchmarks --- store/iavlstore_benchmark_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/iavlstore_benchmark_test.go b/store/iavlstore_benchmark_test.go index e01996c2c3..0d5272d416 100644 --- a/store/iavlstore_benchmark_test.go +++ b/store/iavlstore_benchmark_test.go @@ -39,7 +39,7 @@ var ( type benchFunc func(b require.TestingT, name string) func TestBenchmark(t *testing.T) { - //t.Skip() + t.Skip() log.Setup("debug", "file://-") log.Root.With("module", "diff-iavlstore") testno = 0 From e03d20bf65ec48ee5daad2a6e55e777d46a729e0 Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Mon, 24 Jun 2019 12:22:38 +0100 Subject: [PATCH 23/32] chain iavl revision --- Gopkg.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index 6572f8e83b..4418ed08d3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -306,7 +306,7 @@ name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "ea354ab86e708d9909f6e59099f3a2b2a7dceda4" + revision = "1d05fd16128544e7783c1adc1ed48daa02eafeff" source = "https://github.com/loomnetwork/iavl.git" [[projects]] From 311a07ed455d732e8b237173e92a675d3433a4ec Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 14:39:07 +0700 Subject: [PATCH 24/32] default flushinterval to zero until we have tested in production for some time --- store/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/config.go b/store/config.go index 32006089fd..a9d7ab1c12 100644 --- a/store/config.go +++ b/store/config.go @@ -44,7 +44,7 @@ func DefaultConfig() *AppStoreConfig { NodeCacheSize: 10000, SnapshotVersion: MultiReaderIAVLStoreSnapshotV1, SaveEVMStateToIAVL: false, - IAVLFlushInterval: 100, + IAVLFlushInterval: 0, //default to zero until we know its ready } } From 89000fe9cfd9de3d8624fcb419e47007fa2c7488 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:28:57 +0700 Subject: [PATCH 25/32] use updated IAVL version --- Gopkg.lock | 6 +++--- Gopkg.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 6572f8e83b..a54b5b4815 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -301,12 +301,12 @@ version = "v0.14.0" [[projects]] - branch = "hack1" - digest = "1:fa0ec427da9a605bc7ca7af80e8547c883c0d316b11052f88095993151a8150b" + branch = "master" + digest = "1:fca0b41a76e56d6f15d6d2d8a93529f646acda1734eb8a48c7c9e9611ab1963d" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "ea354ab86e708d9909f6e59099f3a2b2a7dceda4" + revision = "062571e83ae4aed28a0b71daa45c0857b878f36a" source = "https://github.com/loomnetwork/iavl.git" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index bc1f16be15..26fac0ca27 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -79,7 +79,7 @@ ignored = [ [[override]] name = "github.com/tendermint/iavl" source = "https://github.com/loomnetwork/iavl.git" - branch = "hack1" + branch = "master" [[override]] name = "github.com/tendermint/go-amino" From ca631e9afa565f3a4c79093d21e3ce9ab093f907 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:33:13 +0700 Subject: [PATCH 26/32] use updated IAVL version --- Gopkg.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a54b5b4815..b6acbae4c2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -302,11 +302,11 @@ [[projects]] branch = "master" - digest = "1:fca0b41a76e56d6f15d6d2d8a93529f646acda1734eb8a48c7c9e9611ab1963d" + digest = "1:60785defba0e22715577a369c00475446ef9bdca93f0f92c774932f6ff504e6e" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "062571e83ae4aed28a0b71daa45c0857b878f36a" + revision = "16606ddeb69d1a57e2db65dbec29f3e67c7e28f8" source = "https://github.com/loomnetwork/iavl.git" [[projects]] From 0ab00aed051472d0310354a5a5dbde860989c13c Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:44:25 +0700 Subject: [PATCH 27/32] use tmreal on iavl and use HEAD for go-loom --- Gopkg.lock | 4 ++-- Gopkg.toml | 2 +- Makefile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index b6acbae4c2..b3ffb4a4a7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -301,12 +301,12 @@ version = "v0.14.0" [[projects]] - branch = "master" + branch = "tmreal" digest = "1:60785defba0e22715577a369c00475446ef9bdca93f0f92c774932f6ff504e6e" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "16606ddeb69d1a57e2db65dbec29f3e67c7e28f8" + revision = "23f0b990b308f0e932fbd541c1dc77f6ec580fb9" source = "https://github.com/loomnetwork/iavl.git" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index 26fac0ca27..f79c83cf0d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -79,7 +79,7 @@ ignored = [ [[override]] name = "github.com/tendermint/iavl" source = "https://github.com/loomnetwork/iavl.git" - branch = "master" + branch = "tmreal" [[override]] name = "github.com/tendermint/go-amino" diff --git a/Makefile b/Makefile index 94718c3ad3..bdfab4eaf2 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ TRANSFER_GATEWAY_DIR=$(GOPATH)/src/$(PKG_TRANSFER_GATEWAY) # NOTE: To build on Jenkins using a custom go-loom branch update the `deps` target below to checkout # that branch, you only need to update GO_LOOM_GIT_REV if you wish to lock the build to a # specific commit. -GO_LOOM_GIT_REV = 3c6078c4fa4e24e9cc0100204c927c8c674900c8 +GO_LOOM_GIT_REV = HEAD # Specifies the loomnetwork/transfer-gateway branch/revision to use. TG_GIT_REV = HEAD # loomnetwork/go-ethereum loomchain branch From a2772dbbfadb5101e057b6a6e16a1657df6cd3af Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:47:38 +0700 Subject: [PATCH 28/32] use tmreal on iavl and use HEAD for go-loom --- Gopkg.lock | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index b3ffb4a4a7..6859464fc5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -43,17 +43,6 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" -[[projects]] - digest = "1:0594af97b2f4cec6554086eeace6597e20a4b69466eb4ada25adf9f4300dddd2" - name = "github.com/garyburd/redigo" - packages = [ - "internal", - "redis", - ] - pruneopts = "UT" - revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e" - version = "v1.6.0" - [[projects]] digest = "1:4062bc6de62d73e2be342243cf138cf499b34d558876db8d9430e2149388a4d8" name = "github.com/go-logfmt/logfmt" From 618636dcc388984b0ff8189a87ab856989f78157 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:58:09 +0700 Subject: [PATCH 29/32] use tmreal on iavl and use HEAD for go-loom --- Gopkg.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 6859464fc5..c4d25eb0c0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -291,11 +291,11 @@ [[projects]] branch = "tmreal" - digest = "1:60785defba0e22715577a369c00475446ef9bdca93f0f92c774932f6ff504e6e" + digest = "1:9575d2627d57f3c52cbf233e29fd623db8879235bcb6aaacee28822c28bb37e4" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "23f0b990b308f0e932fbd541c1dc77f6ec580fb9" + revision = "43d2409a5e3a3e7911a8e8d460e385764b8a55f1" source = "https://github.com/loomnetwork/iavl.git" [[projects]] From e3f6991751fed53a799246176374835d5d49ad0e Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 3 Jul 2019 12:05:57 +0100 Subject: [PATCH 30/32] remove unit test dependant on nil range --- store/iavlstore_test.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index 1ddf6a872e..8160ca35c3 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -2,7 +2,6 @@ package store import ( "bytes" - "fmt" "os" "testing" @@ -135,16 +134,14 @@ func testNormal(t *testing.T) { diskTree := iavl.NewMutableTree(diskDb, 0) _, err = diskTree.Load() require.NoError(t, err) - fmt.Println("testNormal") - //diskTree.PrintDiskDb() + /* for _, entry := range store.Range(nil) { _, value := tree.Get(entry.Key) require.Zero(t, bytes.Compare(value, entry.Value)) _, diskValue := diskTree.Get(entry.Key) require.Zero(t, bytes.Compare(value, diskValue)) - } - fmt.Println() + }*/ tree.Iterate(func(key []byte, value []byte) bool { require.Zero(t, bytes.Compare(value, store.Get(key))) _, diskValue := diskTree.Get(key) @@ -164,14 +161,13 @@ func testFlush(t *testing.T) { diskTree := iavl.NewMutableTree(diskDb, 0) _, err = diskTree.Load() require.NoError(t, err) - fmt.Println("testFlush") - //diskTree.PrintDiskDb() + /* for _, entry := range store.Range(nil) { _, value := tree.Get(entry.Key) require.Zero(t, bytes.Compare(value, entry.Value)) _, diskValue := diskTree.Get(entry.Key) require.Zero(t, bytes.Compare(value, diskValue)) - } + }*/ tree.Iterate(func(key []byte, value []byte) bool { require.Zero(t, bytes.Compare(value, store.Get(key))) _, diskValue := diskTree.Get(key) @@ -196,15 +192,13 @@ func testMaxVersions(t *testing.T) { diskTree := iavl.NewMutableTree(diskDb, 0) _, err = diskTree.Load() require.NoError(t, err) - - fmt.Println("testMaxVersions") - //diskTree.PrintDiskDb() + /* for _, entry := range store.Range(nil) { _, value := tree.Get(entry.Key) require.Zero(t, bytes.Compare(value, entry.Value)) _, diskValue := diskTree.Get(entry.Key) require.Zero(t, bytes.Compare(value, diskValue)) - } + }*/ tree.Iterate(func(key []byte, value []byte) bool { require.Zero(t, bytes.Compare(value, store.Get(key))) _, diskValue := diskTree.Get(key) From f7d6dbe43f2df042c6390a67f7907e70ffe7da7d Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Wed, 3 Jul 2019 12:12:51 +0100 Subject: [PATCH 31/32] lock --- Gopkg.lock | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 4418ed08d3..a305a9c7e5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -43,17 +43,6 @@ revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" -[[projects]] - digest = "1:0594af97b2f4cec6554086eeace6597e20a4b69466eb4ada25adf9f4300dddd2" - name = "github.com/garyburd/redigo" - packages = [ - "internal", - "redis", - ] - pruneopts = "UT" - revision = "a69d19351219b6dd56f274f96d85a7014a2ec34e" - version = "v1.6.0" - [[projects]] digest = "1:4062bc6de62d73e2be342243cf138cf499b34d558876db8d9430e2149388a4d8" name = "github.com/go-logfmt/logfmt" @@ -301,12 +290,12 @@ version = "v0.14.0" [[projects]] - branch = "hack1" - digest = "1:fa0ec427da9a605bc7ca7af80e8547c883c0d316b11052f88095993151a8150b" + branch = "tmreal" + digest = "1:9575d2627d57f3c52cbf233e29fd623db8879235bcb6aaacee28822c28bb37e4" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "1d05fd16128544e7783c1adc1ed48daa02eafeff" + revision = "43d2409a5e3a3e7911a8e8d460e385764b8a55f1" source = "https://github.com/loomnetwork/iavl.git" [[projects]] @@ -478,4 +467,4 @@ "golang.org/x/sys/cpu", ] solver-name = "gps-cdcl" - solver-version = 1 + solver-version = 1 \ No newline at end of file From 19dcc49d7438ec9e64fe8c5ea18f2f486889fdcc Mon Sep 17 00:00:00 2001 From: Piers Shepperson Date: Fri, 5 Jul 2019 12:33:08 +0100 Subject: [PATCH 32/32] new iavl branch --- Gopkg.lock | 4 ++-- Gopkg.toml | 2 +- store/iavlstore_benchmark_test.go | 31 +------------------------------ store/iavlstore_test.go | 6 +++--- 4 files changed, 7 insertions(+), 36 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a305a9c7e5..7b84b07d21 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -290,12 +290,12 @@ version = "v0.14.0" [[projects]] - branch = "tmreal" + branch = "tmreal2" digest = "1:9575d2627d57f3c52cbf233e29fd623db8879235bcb6aaacee28822c28bb37e4" name = "github.com/tendermint/iavl" packages = ["."] pruneopts = "UT" - revision = "43d2409a5e3a3e7911a8e8d460e385764b8a55f1" + revision = "45ae3144d0e8f894502f09de56de2a65118651fe" source = "https://github.com/loomnetwork/iavl.git" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index 23f78ca135..505df924aa 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -78,7 +78,7 @@ ignored = [ [[override]] name = "github.com/tendermint/iavl" source = "https://github.com/loomnetwork/iavl.git" - branch = "tmreal" + branch = "tmreal2" [[override]] name = "github.com/tendermint/go-amino" diff --git a/store/iavlstore_benchmark_test.go b/store/iavlstore_benchmark_test.go index 0d5272d416..95af292352 100644 --- a/store/iavlstore_benchmark_test.go +++ b/store/iavlstore_benchmark_test.go @@ -56,7 +56,7 @@ func TestBenchmark(t *testing.T) { maxVersions = 2 fmt.Println("numBlocks", numBlocks, "blockSize", blockSize) blocks = nil - blocks = iavl.GenerateBlocks2(numBlocks, blockSize) + blocks = iavl.GenerateBlocksHashKeys(numBlocks, blockSize) tree = iavl.NewMutableTree(db.NewMemDB(), 0) for _, program := range blocks { if err := program.Execute(tree); err != nil { @@ -66,7 +66,6 @@ func TestBenchmark(t *testing.T) { t.Run("normal", benchNormal) t.Run("flush20", benchflush20) - //t.Run("flush100", benchflush100) t.Run("MaxVers", benchMaxVersions) files, err := ioutil.ReadDir("./testdata") @@ -134,12 +133,6 @@ func benchNormal(t *testing.T) { func benchflush20(t *testing.T) { timeIavlStore(t, "flush20", benchmarkflush20) } -func benchFlush50(t *testing.T) { - timeIavlStore(t, "flush50", benchmarkFlush50) -} -func benchflush100(t *testing.T) { - timeIavlStore(t, "flush100", benchmarkflush100) -} func benchMaxVersions(t *testing.T) { timeIavlStore(t, "maxVersions", benchmarkMaxVersions) } @@ -196,28 +189,6 @@ func benchmarkflush20(b require.TestingT, name string) { diskDb.Close() } -func benchmarkFlush50(b require.TestingT, name string) { - testno++ - diskDb := getDiskDb(b, name) - store, err := NewIAVLStore(diskDb, 0, 0, 50) - require.NoError(b, err) - executeBlocks(b, blocks, *store) - _, _, err = store.tree.SaveVersion() - require.NoError(b, err) - diskDb.Close() -} - -func benchmarkflush100(b require.TestingT, name string) { - testno++ - diskDb := getDiskDb(b, name) - store, err := NewIAVLStore(diskDb, 0, 0, 100) - require.NoError(b, err) - executeBlocks(b, blocks, *store) - _, _, err = store.tree.SaveVersion() - require.NoError(b, err) - diskDb.Close() -} - func benchmarkMaxVersions(b require.TestingT, name string) { testno++ diskDb := getDiskDb(b, name) diff --git a/store/iavlstore_test.go b/store/iavlstore_test.go index 8160ca35c3..7540ee8847 100644 --- a/store/iavlstore_test.go +++ b/store/iavlstore_test.go @@ -102,8 +102,8 @@ func testOrphans(t *testing.T, store *IAVLStore, diskDb db.DB, flushInterval int } func TestIavl(t *testing.T) { - numBlocks = 10 - blockSize = 10 + numBlocks = 20 + blockSize = 5 flushInterval = int64(9) log.Setup("debug", "file://-") @@ -114,7 +114,7 @@ func TestIavl(t *testing.T) { require.True(t, os.IsNotExist(err)) blocks = nil - blocks = iavl.GenerateBlocks2(numBlocks, blockSize) + blocks = iavl.GenerateBlocksHashKeys(numBlocks, blockSize) tree = iavl.NewMutableTree(db.NewMemDB(), 0) for _, program := range blocks { require.NoError(t, program.Execute(tree))