Merge PR #6505: Backport v0.38.5: IAVL Bump (RC) & Pruning Refactor

CHANGELOG.md

@@ -35,7 +35,50 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 # Changelog
 
-## [Unreleased]
+## [v0.38.5] - TBD
+
+The pruning changes introduced in 0.38.0 has bugs that may cause data loss, even after upgrading
+to 0.38.5. When upgrading from 0.38.x it is important to follow the instructions below, to prevent 
+data loss and database corruption.
+
+Do not modify pruning settings with <=0.38.4 as that may cause data corruption - the following 
+assumes pruning settings have not been modified since the node started using 0.38.x. The default 
+pruning setting `syncable` used `KeepEvery:100`.
+
+* If using `KeepEvery:1` (pruning settings `nothing` or `everything`), upgrading to 0.38.5 is safe.
+
+* Otherwise, halt block processing with `--halt-height` after committing a height divisible by
+ `KeepEvery` - e.g. at block 147600 with `KeepEvery:100`. The node must _never_ have processed a
+ height beyond that at any time in its past. Upgrading to 0.38.5 is then safe.
+
+* Otherwise, set the 0.38.5 `KeepEvery` setting to the same as the previous `KeepEvery` setting
+ (<=0.38.4 and 0.38.5 both default to `KeepEvery:100`). Upgrading to 0.38.5 is then safe as long
+ as you wait one `KeepEvery` interval plus one `KeepRecent` interval plus one pruning `Interval`
+ before changing pruning settings or deleting the last <=0.38.4 height (so wait 210 heights with
+ the default configuration).
+
+* Otherwise, make sure the last version persisted with <=0.38.4 is never deleted after upgrading to 
+ 0.38.5, as doing so may cause data loss and data corruption.
+
+* Otherwise, consider syncing the node from scratch with 0.38.5.
+
+### Improvements
+
+* (deps) Bump Tendermint version to [v0.33.5](https://github.com/tendermint/tendermint/releases/tag/v0.33.5)
+* (deps) Bump IAVL version to [v0.14.0](https://github.com/cosmos/iavl/releases/tag/v0.14.0)
+
+### Bug Fixes
+
+* (store) Revert IAVL pruning functionality introduced in [v0.13.0](https://github.com/cosmos/iavl/releases/tag/v0.13.0),
+where the IAVL no longer keeps states in-memory in which it flushes periodically. IAVL now commits and
+flushes every state to disk as it did pre-v0.13.0. The SDK's multi-store will track and ensure the proper
+heights are pruned. The operator can set the pruning options via a `pruning` config via the CLI or
+through `app.toml`. The `pruning` flag exposes `default|everything|nothing|custom` as options --
+see docs for further details. If the operator chooses `custom`, they may provide granular pruning
+options `pruning-keep-recent`, `pruning-keep-every`, and `pruning-interval`. The former two options
+dictate how many recent versions are kept on disk and the offset of what versions are kept after that
+respectively, and the latter defines the height interval in which versions are deleted in a batch.
+**Note, there are some client-facing API breaking changes with regard to IAVL, stores, and pruning settings.**
 
 ## [v0.38.4] - 2020-05-21
 

baseapp/baseapp_test.go

@@ -168,7 +168,7 @@ func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) {
 
 func checkStore(t *testing.T, db dbm.DB, ver int64, storeKey string, k, v []byte) {
  rs := rootmulti.NewStore(db)
- rs.SetPruning(store.PruneSyncable)
+ rs.SetPruning(store.PruneDefault)
  key := sdk.NewKVStoreKey(storeKey)
  rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil)
  err := rs.LoadLatestVersion()
@@ -273,7 +273,7 @@ func TestSetLoader(t *testing.T) {
 
 func TestAppVersionSetterGetter(t *testing.T) {
  logger := defaultLogger()
- pruningOpt := SetPruning(store.PruneSyncable)
+ pruningOpt := SetPruning(store.PruneDefault)
  db := dbm.NewMemDB()
  name := t.Name()
  app := NewBaseApp(name, logger, db, nil, pruningOpt)
@@ -329,8 +329,9 @@ func TestLoadVersionInvalid(t *testing.T) {
 func TestLoadVersionPruning(t *testing.T) {
  logger := log.NewNopLogger()
  pruningOptions := store.PruningOptions{
- KeepEvery: 2,
- SnapshotEvery: 6,
+ KeepRecent: 2,
+ KeepEvery: 3,
+ Interval: 1,
  }
  pruningOpt := SetPruning(pruningOptions)
  db := dbm.NewMemDB()
@@ -351,61 +352,33 @@ func TestLoadVersionPruning(t *testing.T) {
  require.Equal(t, int64(0), lastHeight)
  require.Equal(t, emptyCommitID, lastID)
 
- // execute a block
- header := abci.Header{Height: 1}
- app.BeginBlock(abci.RequestBeginBlock{Header: header})
- res := app.Commit()
-
- // execute a block, collect commit ID
- header = abci.Header{Height: 2}
- app.BeginBlock(abci.RequestBeginBlock{Header: header})
- res = app.Commit()
- commitID2 := sdk.CommitID{Version: 2, Hash: res.Data}
-
- // execute a block
- header = abci.Header{Height: 3}
- app.BeginBlock(abci.RequestBeginBlock{Header: header})
- res = app.Commit()
- commitID3 := sdk.CommitID{Version: 3, Hash: res.Data}
-
- // reload with LoadLatestVersion, check it loads last flushed version
- app = NewBaseApp(name, logger, db, nil, pruningOpt)
- app.MountStores(capKey)
- err = app.LoadLatestVersion(capKey)
- require.Nil(t, err)
- testLoadVersionHelper(t, app, int64(2), commitID2)
+ var lastCommitID sdk.CommitID
 
- // re-execute block 3 and check it is same CommitID
- header = abci.Header{Height: 3}
- app.BeginBlock(abci.RequestBeginBlock{Header: header})
- res = app.Commit()
- recommitID3 := sdk.CommitID{Version: 3, Hash: res.Data}
- require.Equal(t, commitID3, recommitID3, "Commits of identical blocks not equal after reload")
+ // Commit seven blocks, of which 7 (latest) is kept in addition to 6, 5
+ // (keep recent) and 3 (keep every).
+ for i := int64(1); i <= 7; i++ {
+ app.BeginBlock(abci.RequestBeginBlock{Header: abci.Header{Height: i}})
+ res := app.Commit()
+ lastCommitID = sdk.CommitID{Version: i, Hash: res.Data}
+ }
 
- // execute a block, collect commit ID
- header = abci.Header{Height: 4}
- app.BeginBlock(abci.RequestBeginBlock{Header: header})
- res = app.Commit()
- commitID4 := sdk.CommitID{Version: 4, Hash: res.Data}
+ for _, v := range []int64{1, 2, 4} {
+ _, err = app.cms.CacheMultiStoreWithVersion(v)
+ require.Error(t, err)
+ }
 
- // execute a block
- header = abci.Header{Height: 5}
- app.BeginBlock(abci.RequestBeginBlock{Header: header})
- res = app.Commit()
+ for _, v := range []int64{3, 5, 6, 7} {
+  _, err = app.cms.CacheMultiStoreWithVersion(v)
+  require.NoError(t, err)
+ }
 
- // reload with LoadLatestVersion, check it loads last flushed version
+ // reload with LoadLatestVersion, check it loads last version
  app = NewBaseApp(name, logger, db, nil, pruningOpt)
  app.MountStores(capKey)
+
  err = app.LoadLatestVersion(capKey)
  require.Nil(t, err)
- testLoadVersionHelper(t, app, int64(4), commitID4)
-
- // reload with LoadVersion of previous flushed version
- // and check it fails since previous flush should be pruned
- app = NewBaseApp(name, logger, db, nil, pruningOpt)
- app.MountStores(capKey)
- err = app.LoadVersion(2, capKey)
- require.NotNil(t, err)
+ testLoadVersionHelper(t, app, int64(7), lastCommitID)
 }
 
 func testLoadVersionHelper(t *testing.T, app *BaseApp, expectedHeight int64, expectedID sdk.CommitID) {

client/context/context.go

@@ -12,6 +12,7 @@ import (
  "github.com/tendermint/tendermint/libs/cli"
  tmlite "github.com/tendermint/tendermint/lite"
  rpcclient "github.com/tendermint/tendermint/rpc/client"
+ rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 
  "github.com/cosmos/cosmos-sdk/client/flags"
  "github.com/cosmos/cosmos-sdk/codec"
@@ -65,7 +66,7 @@ func NewCLIContextWithInputAndFrom(input io.Reader, from string) CLIContext {
  if !genOnly {
  nodeURI = viper.GetString(flags.FlagNode)
  if nodeURI != "" {
- rpc, err = rpcclient.NewHTTP(nodeURI, "/websocket")
+ rpc, err = rpchttp.New(nodeURI, "/websocket")
  if err != nil {
  fmt.Printf("failted to get client: %v\n", err)
  os.Exit(1)
@@ -157,7 +158,7 @@ func (ctx CLIContext) WithTrustNode(trustNode bool) CLIContext {
 // WithNodeURI returns a copy of the context with an updated node URI.
 func (ctx CLIContext) WithNodeURI(nodeURI string) CLIContext {
  ctx.NodeURI = nodeURI
- client, err := rpcclient.NewHTTP(nodeURI, "/websocket")
+ client, err := rpchttp.New(nodeURI, "/websocket")
  if err != nil {
  panic(err)
  }

client/context/verifier.go

@@ -7,7 +7,7 @@ import (
  "github.com/tendermint/tendermint/libs/log"
  tmlite "github.com/tendermint/tendermint/lite"
  tmliteproxy "github.com/tendermint/tendermint/lite/proxy"
- rpcclient "github.com/tendermint/tendermint/rpc/client"
+ rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 )
 
 const (
@@ -43,7 +43,7 @@ func CreateVerifier(ctx CLIContext, cacheSize int) (tmlite.Verifier, error) {
  // create an RPC client based off of the RPC URI if no RPC client exists
  client := ctx.Client
  if client == nil {
- client, err = rpcclient.NewHTTP(ctx.NodeURI, "/websocket")
+ client, err = rpchttp.New(ctx.NodeURI, "/websocket")
  if err != nil {
  return nil, err
  }

client/lcd/root.go

@@ -12,7 +12,7 @@ import (
  "github.com/spf13/cobra"
  "github.com/spf13/viper"
  "github.com/tendermint/tendermint/libs/log"
- rpcserver "github.com/tendermint/tendermint/rpc/lib/server"
+ tmrpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
 
  "github.com/cosmos/cosmos-sdk/client/context"
  "github.com/cosmos/cosmos-sdk/client/flags"
@@ -52,12 +52,12 @@ func (rs *RestServer) Start(listenAddr string, maxOpen int, readTimeout, writeTi
  rs.log.Error("error closing listener", "err", err)
  })
 
- cfg := rpcserver.DefaultConfig()
+ cfg := tmrpcserver.DefaultConfig()
  cfg.MaxOpenConnections = maxOpen
  cfg.ReadTimeout = time.Duration(readTimeout) * time.Second
  cfg.WriteTimeout = time.Duration(writeTimeout) * time.Second
 
- rs.listener, err = rpcserver.Listen(listenAddr, cfg)
+ rs.listener, err = tmrpcserver.Listen(listenAddr, cfg)
  if err != nil {
  return
  }
@@ -68,7 +68,7 @@ func (rs *RestServer) Start(listenAddr string, maxOpen int, readTimeout, writeTi
  ),
  )
 
- return rpcserver.StartHTTPServer(rs.listener, rs.Mux, rs.log, cfg)
+ return tmrpcserver.Serve(rs.listener, rs.Mux, rs.log, cfg)
 }
 
 // ServeCommand will start the application REST service as a blocking process. It

go.mod

@@ -1,10 +1,12 @@
+go 1.13
+
 module github.com/cosmos/cosmos-sdk
 
 require (
  github.com/99designs/keyring v1.1.3
  github.com/bartekn/go-bip39 v0.0.0-20171116152956-a05967ea095d
  github.com/bgentry/speakeasy v0.1.0
- github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d
+ github.com/btcsuite/btcd v0.20.1-beta
  github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d
  github.com/cosmos/ledger-cosmos-go v0.11.1
  github.com/gogo/protobuf v1.3.1
@@ -16,20 +18,19 @@ require (
  github.com/pkg/errors v0.9.1
  github.com/rakyll/statik v0.1.6
  github.com/spf13/afero v1.2.1 // indirect
- github.com/spf13/cobra v0.0.6
+ github.com/spf13/cobra v1.0.0
  github.com/spf13/jwalterweatherman v1.1.0 // indirect
  github.com/spf13/pflag v1.0.5
- github.com/spf13/viper v1.6.2
- github.com/stretchr/testify v1.5.1
+ github.com/spf13/viper v1.6.3
+ github.com/stretchr/testify v1.6.1
  github.com/tendermint/btcd v0.1.1
  github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15
  github.com/tendermint/go-amino v0.15.1
- github.com/tendermint/iavl v0.13.2
- github.com/tendermint/tendermint v0.33.3
- github.com/tendermint/tm-db v0.5.0
- gopkg.in/yaml.v2 v2.2.8
+ github.com/tendermint/iavl v0.14.0-rc2
+ github.com/tendermint/tendermint v0.33.5
+ github.com/tendermint/tm-db v0.5.1
+ google.golang.org/grpc v1.30.0 // indirect
+ gopkg.in/yaml.v2 v2.3.0
 )
 
-go 1.13
-
 replace github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4