Merge PR #4724: Allow substore migrations upon multistore loading

.pending/features/store/4724-Multistore-supp

@@ -0,0 +1,4 @@
+#4724 Multistore supports substore migrations upon load.
+New `rootmulti.Store.LoadLatestVersionAndUpgrade` method
+Baseapp supports `StoreLoader` to enable various upgrade strategies
+No longer panics if the store to load contains substores that we didn't explicitly mount.

baseapp/baseapp.go

@@ -1,8 +1,9 @@
 package baseapp
 
 import (
+	"encoding/json"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os"
 	"reflect"
 	"runtime/debug"
@@ -20,6 +21,7 @@ import (
 
 	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/store"
+	storetypes "github.com/cosmos/cosmos-sdk/store/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
@@ -41,13 +43,20 @@ const (
 	MainStoreKey = "main"
 )
 
+// StoreLoader defines a customizable function to control how we load the CommitMultiStore
+// from disk. This is useful for state migration, when loading a datastore written with
+// an older version of the software. In particular, if a module changed the substore key name
+// (or removed a substore) between two versions of the software.
+type StoreLoader func(ms sdk.CommitMultiStore) error
+
 // BaseApp reflects the ABCI application implementation.
 type BaseApp struct {
 	// initialized on creation
 	logger      log.Logger
 	name        string               // application name from abci.Info
 	db          dbm.DB               // common DB backend
 	cms         sdk.CommitMultiStore // Main (uncached) state
+	storeLoader StoreLoader          // function to handle store loading, may be overridden with SetStoreLoader()
 	router      sdk.Router           // handle any kind of message
 	queryRouter sdk.QueryRouter      // router for redirecting query calls
 	txDecoder   sdk.TxDecoder        // unmarshal []byte into sdk.Tx
@@ -106,6 +115,7 @@ func NewBaseApp(
 		name:           name,
 		db:             db,
 		cms:            store.NewCommitMultiStore(db),
+		storeLoader:    DefaultStoreLoader,
 		router:         NewRouter(),
 		queryRouter:    NewQueryRouter(),
 		txDecoder:      txDecoder,
@@ -133,12 +143,6 @@ func (app *BaseApp) Logger() log.Logger {
 	return app.logger
 }
 
-// SetCommitMultiStoreTracer sets the store tracer on the BaseApp's underlying
-// CommitMultiStore.
-func (app *BaseApp) SetCommitMultiStoreTracer(w io.Writer) {
-	app.cms.SetTracer(w)
-}
-
 // MountStores mounts all IAVL or DB stores to the provided keys in the BaseApp
 // multistore.
 func (app *BaseApp) MountStores(keys ...sdk.StoreKey) {
@@ -197,13 +201,74 @@ func (app *BaseApp) MountStore(key sdk.StoreKey, typ sdk.StoreType) {
 // LoadLatestVersion loads the latest application version. It will panic if
 // called more than once on a running BaseApp.
 func (app *BaseApp) LoadLatestVersion(baseKey *sdk.KVStoreKey) error {
-	err := app.cms.LoadLatestVersion()
+	err := app.storeLoader(app.cms)
 	if err != nil {
 		return err
 	}
 	return app.initFromMainStore(baseKey)
 }
 
+// DefaultStoreLoader will be used by default and loads the latest version
+func DefaultStoreLoader(ms sdk.CommitMultiStore) error {
+	return ms.LoadLatestVersion()
+}
+
+// StoreLoaderWithUpgrade is used to prepare baseapp with a fixed StoreLoader
+// pattern. This is useful in test cases, or with custom upgrade loading logic.
+func StoreLoaderWithUpgrade(upgrades *storetypes.StoreUpgrades) StoreLoader {
+	return func(ms sdk.CommitMultiStore) error {
+		return ms.LoadLatestVersionAndUpgrade(upgrades)
+	}
+}
+
+// UpgradeableStoreLoader can be configured by SetStoreLoader() to check for the
+// existence of a given upgrade file - json encoded StoreUpgrades data.
+//
+// If not file is present, it will peform the default load (no upgrades to store).
+//
+// If the file is present, it will parse the file and execute those upgrades
+// (rename or delete stores), while loading the data. It will also delete the
+// upgrade file upon successful load, so that the upgrade is only applied once,
+// and not re-applied on next restart
+//
+// This is useful for in place migrations when a store key is renamed between
+// two versions of the software. (TODO: this code will move to x/upgrades
+// when PR #4233 is merged, here mainly to help test the design)
+func UpgradeableStoreLoader(upgradeInfoPath string) StoreLoader {
+	return func(ms sdk.CommitMultiStore) error {
+		_, err := os.Stat(upgradeInfoPath)
+		if os.IsNotExist(err) {
+			return DefaultStoreLoader(ms)
+		} else if err != nil {
+			return err
+		}
+
+		// there is a migration file, let's execute
+		data, err := ioutil.ReadFile(upgradeInfoPath)
+		if err != nil {
+			return fmt.Errorf("Cannot read upgrade file %s: %v", upgradeInfoPath, err)
+		}
+
+		var upgrades storetypes.StoreUpgrades
+		err = json.Unmarshal(data, &upgrades)
+		if err != nil {
+			return fmt.Errorf("Cannot parse upgrade file: %v", err)
+		}
+
+		err = ms.LoadLatestVersionAndUpgrade(&upgrades)
+		if err != nil {
+			return fmt.Errorf("Load and upgrade database: %v", err)
+		}
+
+		// if we have a successful load, we delete the file
+		err = os.Remove(upgradeInfoPath)
+		if err != nil {
+			return fmt.Errorf("deleting upgrade file %s: %v", upgradeInfoPath, err)
+		}
+		return nil
+	}
+}
+
 // LoadVersion loads the BaseApp application version. It will panic if called
 // more than once on a running baseapp.
 func (app *BaseApp) LoadVersion(version int64, baseKey *sdk.KVStoreKey) error {

baseapp/baseapp_test.go

@@ -4,11 +4,10 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"testing"
 
-	store "github.com/cosmos/cosmos-sdk/store/types"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -17,6 +16,8 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/cosmos/cosmos-sdk/codec"
+	"github.com/cosmos/cosmos-sdk/store/rootmulti"
+	store "github.com/cosmos/cosmos-sdk/store/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
@@ -130,6 +131,143 @@ func TestLoadVersion(t *testing.T) {
 	testLoadVersionHelper(t, app, int64(2), commitID2)
 }
 
+func useDefaultLoader(app *BaseApp) {
+	app.SetStoreLoader(DefaultStoreLoader)
+}
+
+func useUpgradeLoader(upgrades *store.StoreUpgrades) func(*BaseApp) {
+	return func(app *BaseApp) {
+		app.SetStoreLoader(StoreLoaderWithUpgrade(upgrades))
+	}
+}
+
+func useFileUpgradeLoader(upgradeInfoPath string) func(*BaseApp) {
+	return func(app *BaseApp) {
+		app.SetStoreLoader(UpgradeableStoreLoader(upgradeInfoPath))
+	}
+}
+
+func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) {
+	rs := rootmulti.NewStore(db)
+	rs.SetPruning(store.PruneSyncable)
+	key := sdk.NewKVStoreKey(storeKey)
+	rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil)
+	err := rs.LoadLatestVersion()
+	require.Nil(t, err)
+	require.Equal(t, int64(0), rs.LastCommitID().Version)
+
+	// write some data in substore
+	kv, _ := rs.GetStore(key).(store.KVStore)
+	require.NotNil(t, kv)
+	kv.Set(k, v)
+	commitID := rs.Commit()
+	require.Equal(t, int64(1), commitID.Version)
+}
+
+func checkStore(t *testing.T, db dbm.DB, ver int64, storeKey string, k, v []byte) {
+	rs := rootmulti.NewStore(db)
+	rs.SetPruning(store.PruneSyncable)
+	key := sdk.NewKVStoreKey(storeKey)
+	rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil)
+	err := rs.LoadLatestVersion()
+	require.Nil(t, err)
+	require.Equal(t, ver, rs.LastCommitID().Version)
+
+	// query data in substore
+	kv, _ := rs.GetStore(key).(store.KVStore)
+	require.NotNil(t, kv)
+	require.Equal(t, v, kv.Get(k))
+}
+
+// Test that we can make commits and then reload old versions.
+// Test that LoadLatestVersion actually does.
+func TestSetLoader(t *testing.T) {
+	// write a renamer to a file
+	f, err := ioutil.TempFile("", "upgrade-*.json")
+	require.NoError(t, err)
+	data := []byte(`{"renamed":[{"old_key": "bnk", "new_key": "banker"}]}`)
+	_, err = f.Write(data)
+	require.NoError(t, err)
+	configName := f.Name()
+	require.NoError(t, f.Close())
+
+	// make sure it exists before running everything
+	_, err = os.Stat(configName)
+	require.NoError(t, err)
+
+	cases := map[string]struct {
+		setLoader    func(*BaseApp)
+		origStoreKey string
+		loadStoreKey string
+	}{
+		"don't set loader": {
+			origStoreKey: "foo",
+			loadStoreKey: "foo",
+		},
+		"default loader": {
+			setLoader:    useDefaultLoader,
+			origStoreKey: "foo",
+			loadStoreKey: "foo",
+		},
+		"rename with inline opts": {
+			setLoader: useUpgradeLoader(&store.StoreUpgrades{
+				Renamed: []store.StoreRename{{
+					OldKey: "foo",
+					NewKey: "bar",
+				}},
+			}),
+			origStoreKey: "foo",
+			loadStoreKey: "bar",
+		},
+		"file loader with missing file": {
+			setLoader:    useFileUpgradeLoader(configName + "randomchars"),
+			origStoreKey: "bnk",
+			loadStoreKey: "bnk",
+		},
+		"file loader with existing file": {
+			setLoader:    useFileUpgradeLoader(configName),
+			origStoreKey: "bnk",
+			loadStoreKey: "banker",
+		},
+	}
+
+	k := []byte("key")
+	v := []byte("value")
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			// prepare a db with some data
+			db := dbm.NewMemDB()
+			initStore(t, db, tc.origStoreKey, k, v)
+
+			// load the app with the existing db
+			opts := []func(*BaseApp){SetPruning(store.PruneSyncable)}
+			if tc.setLoader != nil {
+				opts = append(opts, tc.setLoader)
+			}
+			app := NewBaseApp(t.Name(), defaultLogger(), db, nil, opts...)
+			capKey := sdk.NewKVStoreKey(MainStoreKey)
+			app.MountStores(capKey)
+			app.MountStores(sdk.NewKVStoreKey(tc.loadStoreKey))
+			err := app.LoadLatestVersion(capKey)
+			require.Nil(t, err)
+
+			// "execute" one block
+			app.BeginBlock(abci.RequestBeginBlock{Header: abci.Header{Height: 2}})
+			res := app.Commit()
+			require.NotNil(t, res.Data)
+
+			// check db is properly updated
+			checkStore(t, db, 2, tc.loadStoreKey, k, v)
+			checkStore(t, db, 2, tc.loadStoreKey, []byte("foo"), nil)
+		})
+	}
+
+	// ensure config file was deleted
+	_, err = os.Stat(configName)
+	require.True(t, os.IsNotExist(err))
+}
+
 func TestAppVersionSetterGetter(t *testing.T) {
 	logger := defaultLogger()
 	pruningOpt := SetPruning(store.PruneSyncable)
@@ -995,7 +1133,6 @@ func TestMaxBlockGasLimits(t *testing.T) {
 	}
 
 	for i, tc := range testCases {
-		fmt.Printf("debug i: %v\n", i)
 		tx := tc.tx
 
 		// reset the block gas

baseapp/options.go

@@ -3,6 +3,7 @@ package baseapp
 
 import (
 	"fmt"
+	"io"
 
 	dbm "github.com/tendermint/tm-db"
 
@@ -110,3 +111,17 @@ func (app *BaseApp) SetFauxMerkleMode() {
 	}
 	app.fauxMerkleMode = true
 }
+
+// SetCommitMultiStoreTracer sets the store tracer on the BaseApp's underlying
+// CommitMultiStore.
+func (app *BaseApp) SetCommitMultiStoreTracer(w io.Writer) {
+	app.cms.SetTracer(w)
+}
+
+// SetStoreLoader allows us to customize the rootMultiStore initialization.
+func (app *BaseApp) SetStoreLoader(loader StoreLoader) {
+	if app.sealed {
+		panic("SetStoreLoader() on sealed BaseApp")
+	}
+	app.storeLoader = loader
+}

server/mock/store.go

@@ -5,6 +5,7 @@ import (
 
 	dbm "github.com/tendermint/tm-db"
 
+	store "github.com/cosmos/cosmos-sdk/store/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
@@ -70,6 +71,14 @@ func (ms multiStore) LoadLatestVersion() error {
 	return nil
 }
 
+func (ms multiStore) LoadLatestVersionAndUpgrade(upgrades *store.StoreUpgrades) error {
+	return nil
+}
+
+func (ms multiStore) LoadVersionAndUpgrade(ver int64, upgrades *store.StoreUpgrades) error {
+	panic("not implemented")
+}
+
 func (ms multiStore) LoadVersion(ver int64) error {
 	panic("not implemented")
 }