From 0737b226f84e6f7b95c803ee2c88e518022d3fe4 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 16 Feb 2022 08:32:32 -0800 Subject: [PATCH 1/9] start pprof server before load version --- server/start.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/start.go b/server/start.go index f2dfaf246e84..d084218d124f 100644 --- a/server/start.go +++ b/server/start.go @@ -234,6 +234,12 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } } + go func() { + address := "localhost:6060" + ctx.Logger.Info("Starting pprof server", "laddr", address) + ctx.Logger.Error("pprof server error", "err", http.ListenAndServe(address, nil)) + }() + traceWriterFile := ctx.Viper.GetString(flagTraceStore) db, err := openDB(home) if err != nil { From 0ad06a893a62d1af2584128bc5c3fe5e922a59a2 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 16 Feb 2022 08:36:59 -0800 Subject: [PATCH 2/9] net/http/pprof --- simapp/simd/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/simapp/simd/main.go b/simapp/simd/main.go index 3e744360f468..f3831da857a5 100644 --- a/simapp/simd/main.go +++ b/simapp/simd/main.go @@ -7,6 +7,7 @@ import ( svrcmd "github.com/cosmos/cosmos-sdk/server/cmd" "github.com/cosmos/cosmos-sdk/simapp" "github.com/cosmos/cosmos-sdk/simapp/simd/cmd" + _ "net/http/pprof" ) func main() { From 49fd0ca1bcc67e9ef502add1b7c4d650f560dee5 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 18 Feb 2022 07:54:29 -0800 Subject: [PATCH 3/9] v0.17.3-osmo-v2.rc4 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fa650395c514..93f77a297d7e 100644 --- a/go.mod +++ b/go.mod @@ -70,4 +70,4 @@ replace github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0 replace github.com/tendermint/tm-db => github.com/osmosis-labs/tm-db v0.6.5-0.20210911033928-ba9154613417 -replace github.com/cosmos/iavl => github.com/osmosis-labs/iavl v0.17.3-osmo-v1 +replace github.com/cosmos/iavl => github.com/osmosis-labs/iavl v0.17.3-osmo-v2.rc4 diff --git a/go.sum b/go.sum index 5a5b6ebf0b3b..e76bfbee908d 100644 --- a/go.sum +++ b/go.sum @@ -622,6 +622,8 @@ github.com/osmosis-labs/iavl v0.17.3-fast.4 h1:P6872Aq9Q+X2nCQFMpUb+VCP0rNYZPW5O github.com/osmosis-labs/iavl v0.17.3-fast.4/go.mod h1:lJEOIlsd3sVO0JDyXWIXa9/Ur5FBscP26zJx0KxHjto= github.com/osmosis-labs/iavl v0.17.3-osmo-v1 h1:orHUut98Miu2+bsFiNZJ29B3ogrbiBbQpti94L2w3Z4= github.com/osmosis-labs/iavl v0.17.3-osmo-v1/go.mod h1:lJEOIlsd3sVO0JDyXWIXa9/Ur5FBscP26zJx0KxHjto= +github.com/osmosis-labs/iavl v0.17.3-osmo-v2.rc4 h1:3M1NWj1C8Sg0OpIMJiiN2uuxU1ImyL8Nh0ykm1t81e4= +github.com/osmosis-labs/iavl v0.17.3-osmo-v2.rc4/go.mod h1:lJEOIlsd3sVO0JDyXWIXa9/Ur5FBscP26zJx0KxHjto= github.com/osmosis-labs/tm-db v0.6.5-0.20210911033928-ba9154613417 h1:otchJDd2SjFWfs7Tse3ULblGcVWqMJ50BE02XCaqXOo= github.com/osmosis-labs/tm-db v0.6.5-0.20210911033928-ba9154613417/go.mod h1:dptYhIpJ2M5kUuenLr+Yyf3zQOv1SgBZcl8/BmWlMBw= github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ= From 0895f58ac5dc19c562ef94e8064fd511d83b8d24 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 18 Feb 2022 08:03:17 -0800 Subject: [PATCH 4/9] add store key log --- store/iavl/store.go | 7 ++++--- store/iavl/store_test.go | 6 +++--- store/rootmulti/proof_test.go | 2 +- store/rootmulti/store.go | 4 ++-- store/types/iterator_test.go | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/store/iavl/store.go b/store/iavl/store.go index ced9894414ae..39b10911e1d5 100644 --- a/store/iavl/store.go +++ b/store/iavl/store.go @@ -42,15 +42,15 @@ type Store struct { // LoadStore returns an IAVL Store as a CommitKVStore. Internally, it will load the // store's version (id) from the provided DB. An error is returned if the version // fails to load, or if called with a positive version on an empty tree. -func LoadStore(db dbm.DB, logger log.Logger, id types.CommitID, lazyLoading bool) (types.CommitKVStore, error) { - return LoadStoreWithInitialVersion(db, logger, id, lazyLoading, 0) +func LoadStore(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool) (types.CommitKVStore, error) { + return LoadStoreWithInitialVersion(db, logger, key, id, lazyLoading, 0) } // LoadStoreWithInitialVersion returns an IAVL Store as a CommitKVStore setting its initialVersion // to the one given. Internally, it will load the store's version (id) from the // provided DB. An error is returned if the version fails to load, or if called with a positive // version on an empty tree. -func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, id types.CommitID, lazyLoading bool, initialVersion uint64) (types.CommitKVStore, error) { +func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, initialVersion uint64) (types.CommitKVStore, error) { tree, err := iavl.NewMutableTreeWithOpts(db, defaultIAVLCacheSize, &iavl.Options{InitialVersion: initialVersion}) if err != nil { return nil, err @@ -59,6 +59,7 @@ func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, id types.CommitID if tree.IsUpgradeable() && logger != nil { logger.Info( "Upgrading IAVL storage for faster queries + execution on live state. This may take a while", + "store_key", key.String(), "version", initialVersion, "commit", fmt.Sprintf("%X", id), "is_lazy", lazyLoading, diff --git a/store/iavl/store_test.go b/store/iavl/store_test.go index 3f635a707129..06185506a907 100644 --- a/store/iavl/store_test.go +++ b/store/iavl/store_test.go @@ -94,17 +94,17 @@ func TestLoadStore(t *testing.T) { require.Equal(t, string(hcStore.Get([]byte("hello"))), "ciao") // Querying a new store at some previous non-pruned height H - newHStore, err := LoadStore(db, log.NewNopLogger(), cIDH, false) + newHStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDH, false) require.NoError(t, err) require.Equal(t, string(newHStore.Get([]byte("hello"))), "hallo") // Querying a new store at some previous pruned height Hp - newHpStore, err := LoadStore(db, log.NewNopLogger(), cIDHp, false) + newHpStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHp, false) require.NoError(t, err) require.Equal(t, string(newHpStore.Get([]byte("hello"))), "hola") // Querying a new store at current height H - newHcStore, err := LoadStore(db, log.NewNopLogger(), cIDHc, false) + newHcStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHc, false) require.NoError(t, err) require.Equal(t, string(newHcStore.Get([]byte("hello"))), "ciao") } diff --git a/store/rootmulti/proof_test.go b/store/rootmulti/proof_test.go index 2c301a3b2ac9..de557b3ef96d 100644 --- a/store/rootmulti/proof_test.go +++ b/store/rootmulti/proof_test.go @@ -15,7 +15,7 @@ import ( func TestVerifyIAVLStoreQueryProof(t *testing.T) { // Create main tree for testing. db := dbm.NewMemDB() - iStore, err := iavl.LoadStore(db, nil, types.CommitID{}, false) + iStore, err := iavl.LoadStore(db, nil, types.NewKVStoreKey("test"), types.CommitID{}, false) store := iStore.(*iavl.Store) require.Nil(t, err) store.Set([]byte("MYKEY"), []byte("MYVALUE")) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 3a8e2592f19e..84ea001004e5 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -879,9 +879,9 @@ func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID var err error if params.initialVersion == 0 { - store, err = iavl.LoadStore(db, rs.logger, id, rs.lazyLoading) + store, err = iavl.LoadStore(db, rs.logger, key, id, rs.lazyLoading) } else { - store, err = iavl.LoadStoreWithInitialVersion(db, rs.logger, id, rs.lazyLoading, params.initialVersion) + store, err = iavl.LoadStoreWithInitialVersion(db, rs.logger, key, id, rs.lazyLoading, params.initialVersion) } if err != nil { diff --git a/store/types/iterator_test.go b/store/types/iterator_test.go index 942983683c9e..9d60d6fd4c1e 100644 --- a/store/types/iterator_test.go +++ b/store/types/iterator_test.go @@ -13,7 +13,7 @@ import ( func newMemTestKVStore(t *testing.T) types.KVStore { db := dbm.NewMemDB() - store, err := iavl.LoadStore(db, log.NewNopLogger(), types.CommitID{}, false) + store, err := iavl.LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), types.CommitID{}, false) require.NoError(t, err) return store } From 7ec05684856bf87b868f67d05459d64931ac599d Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 18 Feb 2022 18:05:34 -0800 Subject: [PATCH 5/9] flush commit metadata when prunning fails --- store/rootmulti/store.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 84ea001004e5..2ff1af94c7e8 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -382,6 +382,14 @@ func (rs *Store) Commit() types.CommitID { rs.lastCommitInfo = commitStores(version, rs.stores) + var err error + defer func () { + flushMetadata(rs.db, version, rs.lastCommitInfo, rs.pruneHeights) + if err != nil { + panic(err) + } + }() + // Determine if pruneHeight height needs to be added to the list of heights to // be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent. if int64(rs.pruningOpts.KeepRecent) < previousHeight { @@ -398,11 +406,9 @@ func (rs *Store) Commit() types.CommitID { // batch prune if the current height is a pruning interval height if rs.pruningOpts.Interval > 0 && version%int64(rs.pruningOpts.Interval) == 0 { - rs.pruneStores() + err = rs.pruneStores() } - flushMetadata(rs.db, version, rs.lastCommitInfo, rs.pruneHeights) - return types.CommitID{ Version: version, Hash: rs.lastCommitInfo.Hash(), @@ -411,9 +417,9 @@ func (rs *Store) Commit() types.CommitID { // pruneStores will batch delete a list of heights from each mounted sub-store. // Afterwards, pruneHeights is reset. -func (rs *Store) pruneStores() { +func (rs *Store) pruneStores() error { if len(rs.pruneHeights) == 0 { - return + return nil } for key, store := range rs.stores { @@ -424,13 +430,14 @@ func (rs *Store) pruneStores() { if err := store.(*iavl.Store).DeleteVersions(rs.pruneHeights...); err != nil { if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist { - panic(err) + return err } } } } rs.pruneHeights = make([]int64, 0) + return nil } // CacheWrap implements CacheWrapper/Store/CommitStore. From f3085d87d507f24e739e2e05ca65fb270194f684 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 18 Feb 2022 19:53:31 -0800 Subject: [PATCH 6/9] rename err to pruneErr --- store/rootmulti/store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 2ff1af94c7e8..6de1c419c35c 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -382,11 +382,11 @@ func (rs *Store) Commit() types.CommitID { rs.lastCommitInfo = commitStores(version, rs.stores) - var err error + var pruneErr error defer func () { flushMetadata(rs.db, version, rs.lastCommitInfo, rs.pruneHeights) - if err != nil { - panic(err) + if pruneErr != nil { + panic(pruneErr) } }() @@ -406,7 +406,7 @@ func (rs *Store) Commit() types.CommitID { // batch prune if the current height is a pruning interval height if rs.pruningOpts.Interval > 0 && version%int64(rs.pruningOpts.Interval) == 0 { - err = rs.pruneStores() + pruneErr = rs.pruneStores() } return types.CommitID{ From ce47713d87057022a15daa691f9ac4993de4563e Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 19 Feb 2022 17:49:02 -0800 Subject: [PATCH 7/9] remove pprof server from start.go --- server/start.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/start.go b/server/start.go index d084218d124f..f2dfaf246e84 100644 --- a/server/start.go +++ b/server/start.go @@ -234,12 +234,6 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } } - go func() { - address := "localhost:6060" - ctx.Logger.Info("Starting pprof server", "laddr", address) - ctx.Logger.Error("pprof server error", "err", http.ListenAndServe(address, nil)) - }() - traceWriterFile := ctx.Viper.GetString(flagTraceStore) db, err := openDB(home) if err != nil { From 690aaa5bcc610013b42f80881c64d13a10a8ae24 Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 19 Feb 2022 21:06:11 -0800 Subject: [PATCH 8/9] upgrade iavl --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 93f77a297d7e..efeabf413ac8 100644 --- a/go.mod +++ b/go.mod @@ -70,4 +70,4 @@ replace github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0 replace github.com/tendermint/tm-db => github.com/osmosis-labs/tm-db v0.6.5-0.20210911033928-ba9154613417 -replace github.com/cosmos/iavl => github.com/osmosis-labs/iavl v0.17.3-osmo-v2.rc4 +replace github.com/cosmos/iavl => github.com/osmosis-labs/iavl v0.17.3-osmo-v3 diff --git a/go.sum b/go.sum index e76bfbee908d..7d0490d0f1b3 100644 --- a/go.sum +++ b/go.sum @@ -624,6 +624,8 @@ github.com/osmosis-labs/iavl v0.17.3-osmo-v1 h1:orHUut98Miu2+bsFiNZJ29B3ogrbiBbQ github.com/osmosis-labs/iavl v0.17.3-osmo-v1/go.mod h1:lJEOIlsd3sVO0JDyXWIXa9/Ur5FBscP26zJx0KxHjto= github.com/osmosis-labs/iavl v0.17.3-osmo-v2.rc4 h1:3M1NWj1C8Sg0OpIMJiiN2uuxU1ImyL8Nh0ykm1t81e4= github.com/osmosis-labs/iavl v0.17.3-osmo-v2.rc4/go.mod h1:lJEOIlsd3sVO0JDyXWIXa9/Ur5FBscP26zJx0KxHjto= +github.com/osmosis-labs/iavl v0.17.3-osmo-v3 h1:q2Qv3+DK52w5b68I96VApHI05jBu7HeQK/YDttKh/jY= +github.com/osmosis-labs/iavl v0.17.3-osmo-v3/go.mod h1:lJEOIlsd3sVO0JDyXWIXa9/Ur5FBscP26zJx0KxHjto= github.com/osmosis-labs/tm-db v0.6.5-0.20210911033928-ba9154613417 h1:otchJDd2SjFWfs7Tse3ULblGcVWqMJ50BE02XCaqXOo= github.com/osmosis-labs/tm-db v0.6.5-0.20210911033928-ba9154613417/go.mod h1:dptYhIpJ2M5kUuenLr+Yyf3zQOv1SgBZcl8/BmWlMBw= github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ= From 70abe6fa262202ebef98097133c94be3c19c2a42 Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 19 Feb 2022 21:11:48 -0800 Subject: [PATCH 9/9] remove pprof --- simapp/simd/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/simapp/simd/main.go b/simapp/simd/main.go index f3831da857a5..3e744360f468 100644 --- a/simapp/simd/main.go +++ b/simapp/simd/main.go @@ -7,7 +7,6 @@ import ( svrcmd "github.com/cosmos/cosmos-sdk/server/cmd" "github.com/cosmos/cosmos-sdk/simapp" "github.com/cosmos/cosmos-sdk/simapp/simd/cmd" - _ "net/http/pprof" ) func main() {