From dfc8dd813e041dfb21cc188ad597522f19a75927 Mon Sep 17 00:00:00 2001 From: Amaury Date: Wed, 10 Feb 2021 18:49:31 +0100 Subject: [PATCH] Add in-place store migrations (#8485) * Add 1st version of migrate * Put migration logic into Configurator * add test to bank store migration * add test for configurator * Error if no migration found * Remove RunMigrations from Configurator interface * Update spec * Rename folders * copy-paste from keys.go * Fix nil map * rename function * Update simapp/app.go Co-authored-by: Robert Zaremba * Update simapp/app_test.go Co-authored-by: Robert Zaremba * Adderss reviews * Fix tests * Update testutil/context.go Co-authored-by: Robert Zaremba * Update docs for ConsensusVersion * Rename to forVersion * Fix tests * Check error early * Return 1 for intiial version * Use MigrationKeeper * Fix test * Revert adding marshaler to Configurator * Godoc updates * Update docs Co-authored-by: Robert Zaremba Co-authored-by: Aaron Craelius --- simapp/app.go | 28 ++++++++- simapp/app_test.go | 89 ++++++++++++++++++++++++++- tests/mocks/types_module_module.go | 3 + testutil/context.go | 25 ++++++++ types/context_test.go | 17 +---- types/module/configurator.go | 75 +++++++++++++++++++++- types/module/module.go | 34 ++++++++++ x/auth/legacy/v040/store.go | 4 ++ x/auth/module.go | 3 + x/auth/vesting/module.go | 3 + x/authz/module.go | 3 + x/bank/keeper/migrations.go | 20 ++++++ x/bank/legacy/v040/keys.go | 47 ++++++++++++++ x/bank/legacy/v040/types.go | 5 -- x/bank/legacy/v042/keys.go | 50 +++++++++++++++ x/bank/legacy/v042/store.go | 38 ++++++++++++ x/bank/legacy/v042/store_test.go | 36 +++++++++++ x/bank/module.go | 6 ++ x/bank/spec/01_state.md | 2 +- x/capability/module.go | 3 + x/crisis/module.go | 3 + x/distribution/module.go | 3 + x/evidence/module.go | 3 + x/feegrant/module.go | 3 + x/genutil/module.go | 3 + x/gov/module.go | 3 + x/ibc/applications/transfer/module.go | 3 + x/ibc/core/module.go | 3 + x/mint/module.go | 3 + x/params/keeper/common_test.go | 24 +------- x/params/module.go | 3 + x/slashing/module.go | 3 + x/staking/module.go | 3 + x/upgrade/module.go | 3 + 34 files changed, 509 insertions(+), 45 deletions(-) create mode 100644 testutil/context.go create mode 100644 x/auth/legacy/v040/store.go create mode 100644 x/bank/keeper/migrations.go create mode 100644 x/bank/legacy/v040/keys.go delete mode 100644 x/bank/legacy/v040/types.go create mode 100644 x/bank/legacy/v042/keys.go create mode 100644 x/bank/legacy/v042/store.go create mode 100644 x/bank/legacy/v042/store_test.go diff --git a/simapp/app.go b/simapp/app.go index 23ced043d047..4db7d53d7755 100644 --- a/simapp/app.go +++ b/simapp/app.go @@ -198,6 +198,9 @@ type SimApp struct { // simulation manager sm *module.SimulationManager + + // the configurator + configurator module.Configurator } func init() { @@ -393,7 +396,8 @@ func NewSimApp( app.mm.RegisterInvariants(&app.CrisisKeeper) app.mm.RegisterRoutes(app.Router(), app.QueryRouter(), encodingConfig.Amino) - app.mm.RegisterServices(module.NewConfigurator(app.MsgServiceRouter(), app.GRPCQueryRouter())) + app.configurator = module.NewConfigurator(app.MsgServiceRouter(), app.GRPCQueryRouter()) + app.mm.RegisterServices(app.configurator) // add test gRPC service for testing gRPC queries in isolation testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) @@ -598,6 +602,28 @@ func (app *SimApp) RegisterTendermintService(clientCtx client.Context) { tmservice.RegisterTendermintService(app.BaseApp.GRPCQueryRouter(), clientCtx, app.interfaceRegistry) } +// RunMigrations performs in-place store migrations for all modules. This +// function MUST be only called by x/upgrade UpgradeHandler. +// +// `migrateFromVersions` is a map of moduleName to fromVersion (unit64), where +// fromVersion denotes the version from which we should migrate the module, the +// target version being the module's latest ConsensusVersion. +// +// Example: +// cfg := module.NewConfigurator(...) +// app.UpgradeKeeper.SetUpgradeHandler("store-migration", func(ctx sdk.Context, plan upgradetypes.Plan) { +// err := app.RunMigrations(ctx, module.MigrationMap{ +// "bank": 1, // Migrate x/bank from v1 to current x/bank's ConsensusVersion +// "staking": 8, // Migrate x/staking from v8 to current x/staking's ConsensusVersion +// }) +// if err != nil { +// panic(err) +// } +// }) +func (app *SimApp) RunMigrations(ctx sdk.Context, migrateFromVersions module.MigrationMap) error { + return app.mm.RunMigrations(ctx, app.configurator, migrateFromVersions) +} + // RegisterSwaggerAPI registers swagger route with API Server func RegisterSwaggerAPI(ctx client.Context, rtr *mux.Router) { statikFS, err := fs.New() diff --git a/simapp/app_test.go b/simapp/app_test.go index 6543f94fd437..2adcf2e7e8d2 100644 --- a/simapp/app_test.go +++ b/simapp/app_test.go @@ -6,10 +6,13 @@ import ( "testing" "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/log" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" dbm "github.com/tendermint/tm-db" - abci "github.com/tendermint/tendermint/abci/types" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/module" ) func TestSimAppExportAndBlockedAddrs(t *testing.T) { @@ -45,3 +48,87 @@ func TestGetMaccPerms(t *testing.T) { dup := GetMaccPerms() require.Equal(t, maccPerms, dup, "duplicated module account permissions differed from actual module account permissions") } + +func TestRunMigrations(t *testing.T) { + db := dbm.NewMemDB() + encCfg := MakeTestEncodingConfig() + app := NewSimApp(log.NewTMLogger(log.NewSyncWriter(os.Stdout)), db, nil, true, map[int64]bool{}, DefaultNodeHome, 0, encCfg, EmptyAppOptions{}) + + // Create a new configurator for the purpose of this test. + app.configurator = module.NewConfigurator(app.MsgServiceRouter(), app.GRPCQueryRouter()) + + testCases := []struct { + name string + moduleName string + forVersion uint64 + expRegErr bool // errors while registering migration + expRegErrMsg string + expRunErr bool // errors while running migration + expRunErrMsg string + expCalled int + }{ + { + "cannot register migration for version 0", + "bank", 0, + true, "module migration versions should start at 1: invalid version", false, "", 0, + }, + { + "throws error on RunMigrations if no migration registered for bank", + "", 1, + false, "", true, "no migrations found for module bank: not found", 0, + }, + { + "can register and run migration handler for x/bank", + "bank", 1, + false, "", false, "", 1, + }, + { + "cannot register migration handler for same module & forVersion", + "bank", 1, + true, "another migration for module bank and version 1 already exists: internal logic error", false, "", 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var err error + + // Since it's very hard to test actual in-place store migrations in + // tests (due to the difficulty of maintaing multiple versions of a + // module), we're just testing here that the migration logic is + // called. + called := 0 + + if tc.moduleName != "" { + // Register migration for module from version `forVersion` to `forVersion+1`. + err = app.configurator.RegisterMigration(tc.moduleName, tc.forVersion, func(sdk.Context) error { + called++ + + return nil + }) + + if tc.expRegErr { + require.EqualError(t, err, tc.expRegErrMsg) + + return + } + } + require.NoError(t, err) + + err = app.RunMigrations( + app.NewContext(true, tmproto.Header{Height: app.LastBlockHeight()}), + module.MigrationMap{ + "auth": 1, "authz": 1, "bank": 1, "staking": 1, "mint": 1, "distribution": 1, + "slashing": 1, "gov": 1, "params": 1, "ibc": 1, "upgrade": 1, "vesting": 1, + "feegrant": 1, "transfer": 1, "evidence": 1, "crisis": 1, "genutil": 1, "capability": 1, + }, + ) + if tc.expRunErr { + require.EqualError(t, err, tc.expRunErrMsg) + } else { + require.NoError(t, err) + require.Equal(t, tc.expCalled, called) + } + }) + } +} diff --git a/tests/mocks/types_module_module.go b/tests/mocks/types_module_module.go index 41ded4d7a2c0..0d00584ab9a8 100644 --- a/tests/mocks/types_module_module.go +++ b/tests/mocks/types_module_module.go @@ -492,6 +492,9 @@ func (m *MockAppModule) ExportGenesis(arg0 types0.Context, arg1 codec.JSONMarsha return ret0 } +// ConsensusVersion mocks base method +func (m *MockAppModule) ConsensusVersion() uint64 { return 1 } + // ExportGenesis indicates an expected call of ExportGenesis func (mr *MockAppModuleMockRecorder) ExportGenesis(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() diff --git a/testutil/context.go b/testutil/context.go new file mode 100644 index 000000000000..2fb9865a2696 --- /dev/null +++ b/testutil/context.go @@ -0,0 +1,25 @@ +package testutil + +import ( + "github.com/tendermint/tendermint/libs/log" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + dbm "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/store" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +// DefaultContext creates a sdk.Context with a fresh MemDB that can be used in tests. +func DefaultContext(key sdk.StoreKey, tkey sdk.StoreKey) sdk.Context { + db := dbm.NewMemDB() + cms := store.NewCommitMultiStore(db) + cms.MountStoreWithDB(key, sdk.StoreTypeIAVL, db) + cms.MountStoreWithDB(tkey, sdk.StoreTypeTransient, db) + err := cms.LoadLatestVersion() + if err != nil { + panic(err) + } + ctx := sdk.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger()) + + return ctx +} diff --git a/types/context_test.go b/types/context_test.go index 018bd6a25792..4af5d8390f2f 100644 --- a/types/context_test.go +++ b/types/context_test.go @@ -8,13 +8,11 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/suite" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/libs/log" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - "github.com/cosmos/cosmos-sdk/store" "github.com/cosmos/cosmos-sdk/tests/mocks" + "github.com/cosmos/cosmos-sdk/testutil" "github.com/cosmos/cosmos-sdk/types" ) @@ -26,15 +24,6 @@ func TestContextTestSuite(t *testing.T) { suite.Run(t, new(contextTestSuite)) } -func (s *contextTestSuite) defaultContext(key types.StoreKey) types.Context { - db := dbm.NewMemDB() - cms := store.NewCommitMultiStore(db) - cms.MountStoreWithDB(key, types.StoreTypeIAVL, db) - s.Require().NoError(cms.LoadLatestVersion()) - ctx := types.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger()) - return ctx -} - func (s *contextTestSuite) TestCacheContext() { key := types.NewKVStoreKey(s.T().Name() + "_TestCacheContext") k1 := []byte("hello") @@ -42,7 +31,7 @@ func (s *contextTestSuite) TestCacheContext() { k2 := []byte("key") v2 := []byte("value") - ctx := s.defaultContext(key) + ctx := testutil.DefaultContext(key, types.NewTransientStoreKey("transient_"+s.T().Name())) store := ctx.KVStore(key) store.Set(k1, v1) s.Require().Equal(v1, store.Get(k1)) @@ -64,7 +53,7 @@ func (s *contextTestSuite) TestCacheContext() { func (s *contextTestSuite) TestLogContext() { key := types.NewKVStoreKey(s.T().Name()) - ctx := s.defaultContext(key) + ctx := testutil.DefaultContext(key, types.NewTransientStoreKey("transient_"+s.T().Name())) ctrl := gomock.NewController(s.T()) s.T().Cleanup(ctrl.Finish) diff --git a/types/module/configurator.go b/types/module/configurator.go index d561dd9eef0d..0e766df2a13d 100644 --- a/types/module/configurator.go +++ b/types/module/configurator.go @@ -1,6 +1,11 @@ package module -import "github.com/gogo/protobuf/grpc" +import ( + "github.com/gogo/protobuf/grpc" + + sdk "github.com/cosmos/cosmos-sdk/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" +) // Configurator provides the hooks to allow modules to configure and register // their services in the RegisterServices method. It is designed to eventually @@ -15,16 +20,34 @@ type Configurator interface { // QueryServer returns a grpc.Server instance which allows registering services // that will be exposed as gRPC services as well as ABCI query handlers. QueryServer() grpc.Server + + // RegisterMigration registers an in-place store migration for a module. The + // handler is a migration script to perform in-place migrations from version + // `forVersion` to version `forVersion+1`. + // + // EACH TIME a module's ConsensusVersion increments, a new migration MUST + // be registered using this function. If a migration handler is missing for + // a particular function, the upgrade logic (see RunMigrations function) + // will panic. If the ConsensusVersion bump does not introduce any store + // changes, then a no-op function must be registered here. + RegisterMigration(moduleName string, forVersion uint64, handler MigrationHandler) error } type configurator struct { msgServer grpc.Server queryServer grpc.Server + + // migrations is a map of moduleName -> forVersion -> migration script handler + migrations map[string]map[uint64]MigrationHandler } // NewConfigurator returns a new Configurator instance func NewConfigurator(msgServer grpc.Server, queryServer grpc.Server) Configurator { - return configurator{msgServer: msgServer, queryServer: queryServer} + return configurator{ + msgServer: msgServer, + queryServer: queryServer, + migrations: map[string]map[uint64]MigrationHandler{}, + } } var _ Configurator = configurator{} @@ -38,3 +61,51 @@ func (c configurator) MsgServer() grpc.Server { func (c configurator) QueryServer() grpc.Server { return c.queryServer } + +// RegisterMigration implements the Configurator.RegisterMigration method +func (c configurator) RegisterMigration(moduleName string, forVersion uint64, handler MigrationHandler) error { + if forVersion == 0 { + return sdkerrors.Wrap(sdkerrors.ErrInvalidVersion, "module migration versions should start at 1") + } + + if c.migrations[moduleName] == nil { + c.migrations[moduleName] = map[uint64]MigrationHandler{} + } + + if c.migrations[moduleName][forVersion] != nil { + return sdkerrors.Wrapf(sdkerrors.ErrLogic, "another migration for module %s and version %d already exists", moduleName, forVersion) + } + + c.migrations[moduleName][forVersion] = handler + + return nil +} + +// runModuleMigrations runs all in-place store migrations for one given module from a +// version to another version. +func (c configurator) runModuleMigrations(ctx sdk.Context, moduleName string, fromVersion, toVersion uint64) error { + // No-op if toVersion is the initial version. + if toVersion <= 1 { + return nil + } + + moduleMigrationsMap, found := c.migrations[moduleName] + if !found { + return sdkerrors.Wrapf(sdkerrors.ErrNotFound, "no migrations found for module %s", moduleName) + } + + // Run in-place migrations for the module sequentially until toVersion. + for i := fromVersion; i < toVersion; i++ { + migrateFn, found := moduleMigrationsMap[i] + if !found { + return sdkerrors.Wrapf(sdkerrors.ErrNotFound, "no migration found for module %s from version %d to version %d", moduleName, i, i+1) + } + + err := migrateFn(ctx) + if err != nil { + return err + } + } + + return nil +} diff --git a/types/module/module.go b/types/module/module.go index 2379c93d5ebd..58bb4927b29a 100644 --- a/types/module/module.go +++ b/types/module/module.go @@ -40,6 +40,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" sdk "github.com/cosmos/cosmos-sdk/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) //__________________________________________________________________________________________ @@ -174,6 +175,12 @@ type AppModule interface { // RegisterServices allows a module to register services RegisterServices(Configurator) + // ConsensusVersion is a sequence number for state-breaking change of the + // module. It should be incremented on each consensus-breaking change + // introduced by the module. To avoid wrong/empty versions, the initial version + // should be set to 1. + ConsensusVersion() uint64 + // ABCI BeginBlock(sdk.Context, abci.RequestBeginBlock) EndBlock(sdk.Context, abci.RequestEndBlock) []abci.ValidatorUpdate @@ -208,6 +215,9 @@ func (gam GenesisOnlyAppModule) LegacyQuerierHandler(*codec.LegacyAmino) sdk.Que // RegisterServices registers all services. func (gam GenesisOnlyAppModule) RegisterServices(Configurator) {} +// ConsensusVersion implements AppModule/ConsensusVersion. +func (gam GenesisOnlyAppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns an empty module begin-block func (gam GenesisOnlyAppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) {} @@ -328,6 +338,30 @@ func (m *Manager) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) map[st return genesisData } +// MigrationHandler is the migration function that each module registers. +type MigrationHandler func(store sdk.Context) error + +// MigrationMap is a map of moduleName -> version, where version denotes the +// version from which we should perform the migration for each module. +type MigrationMap map[string]uint64 + +// RunMigrations performs in-place store migrations for all modules. +func (m Manager) RunMigrations(ctx sdk.Context, cfg Configurator, migrateFromVersions MigrationMap) error { + c, ok := cfg.(configurator) + if !ok { + return sdkerrors.Wrapf(sdkerrors.ErrInvalidType, "expected %T, got %T", configurator{}, cfg) + } + + for moduleName, module := range m.Modules { + err := c.runModuleMigrations(ctx, moduleName, migrateFromVersions[moduleName], module.ConsensusVersion()) + if err != nil { + return err + } + } + + return nil +} + // BeginBlock performs begin block functionality for all modules. It creates a // child context with an event manager to aggregate events emitted from all // modules. diff --git a/x/auth/legacy/v040/store.go b/x/auth/legacy/v040/store.go new file mode 100644 index 000000000000..9fb81b60a749 --- /dev/null +++ b/x/auth/legacy/v040/store.go @@ -0,0 +1,4 @@ +package v040 + +// AddrLen defines a valid address length +const AddrLen = 20 diff --git a/x/auth/module.go b/x/auth/module.go index 73aa9a1066a9..9e4af71ff037 100644 --- a/x/auth/module.go +++ b/x/auth/module.go @@ -147,6 +147,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the auth module. func (AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/auth/vesting/module.go b/x/auth/vesting/module.go index 3cc579a40e37..259143469388 100644 --- a/x/auth/vesting/module.go +++ b/x/auth/vesting/module.go @@ -127,3 +127,6 @@ func (am AppModule) EndBlock(_ sdk.Context, _ abci.RequestEndBlock) []abci.Valid func (am AppModule) ExportGenesis(_ sdk.Context, cdc codec.JSONMarshaler) json.RawMessage { return am.DefaultGenesis(cdc) } + +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } diff --git a/x/authz/module.go b/x/authz/module.go index 1bcd388e4df9..b39c02487fe0 100644 --- a/x/authz/module.go +++ b/x/authz/module.go @@ -152,6 +152,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) {} // EndBlock does nothing diff --git a/x/bank/keeper/migrations.go b/x/bank/keeper/migrations.go new file mode 100644 index 000000000000..de6b61a9c2b6 --- /dev/null +++ b/x/bank/keeper/migrations.go @@ -0,0 +1,20 @@ +package keeper + +import ( + sdk "github.com/cosmos/cosmos-sdk/types" + v042 "github.com/cosmos/cosmos-sdk/x/bank/legacy/v042" +) + +// MigrationKeeper is an interface that the keeper implements for handling +// in-place store migrations. +type MigrationKeeper interface { + // Migrate1 migrates the store from version 1 to 2. + Migrate1(ctx sdk.Context) error +} + +var _ MigrationKeeper = (*BaseKeeper)(nil) + +// Migrate1 implements MigrationKeeper.Migrate1 method. +func (keeper BaseKeeper) Migrate1(ctx sdk.Context) error { + return v042.MigrateStore(ctx, keeper.storeKey) +} diff --git a/x/bank/legacy/v040/keys.go b/x/bank/legacy/v040/keys.go new file mode 100644 index 000000000000..bd1035599164 --- /dev/null +++ b/x/bank/legacy/v040/keys.go @@ -0,0 +1,47 @@ +package v040 + +import ( + "fmt" + + sdk "github.com/cosmos/cosmos-sdk/types" + v040auth "github.com/cosmos/cosmos-sdk/x/auth/legacy/v040" +) + +const ( + // ModuleName defines the module name + ModuleName = "bank" + + // StoreKey defines the primary module store key + StoreKey = ModuleName + + // RouterKey defines the module's message routing key + RouterKey = ModuleName + + // QuerierRoute defines the module's query routing key + QuerierRoute = ModuleName +) + +// KVStore keys +var ( + BalancesPrefix = []byte("balances") + SupplyKey = []byte{0x00} + DenomMetadataPrefix = []byte{0x1} +) + +// DenomMetadataKey returns the denomination metadata key. +func DenomMetadataKey(denom string) []byte { + d := []byte(denom) + return append(DenomMetadataPrefix, d...) +} + +// AddressFromBalancesStore returns an account address from a balances prefix +// store. The key must not contain the perfix BalancesPrefix as the prefix store +// iterator discards the actual prefix. +func AddressFromBalancesStore(key []byte) sdk.AccAddress { + addr := key[:v040auth.AddrLen] + if len(addr) != v040auth.AddrLen { + panic(fmt.Sprintf("unexpected account address key length; got: %d, expected: %d", len(addr), v040auth.AddrLen)) + } + + return sdk.AccAddress(addr) +} diff --git a/x/bank/legacy/v040/types.go b/x/bank/legacy/v040/types.go deleted file mode 100644 index 04ac05f9fcd3..000000000000 --- a/x/bank/legacy/v040/types.go +++ /dev/null @@ -1,5 +0,0 @@ -package v040 - -const ( - ModuleName = "bank" -) diff --git a/x/bank/legacy/v042/keys.go b/x/bank/legacy/v042/keys.go new file mode 100644 index 000000000000..8d718b1c5d3c --- /dev/null +++ b/x/bank/legacy/v042/keys.go @@ -0,0 +1,50 @@ +package v042 + +import ( + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/address" +) + +const ( + // ModuleName defines the module name + ModuleName = "bank" + + // StoreKey defines the primary module store key + StoreKey = ModuleName + + // RouterKey defines the module's message routing key + RouterKey = ModuleName + + // QuerierRoute defines the module's query routing key + QuerierRoute = ModuleName +) + +// KVStore keys +var ( + // BalancesPrefix is the for the account balances store. We use a byte + // (instead of say `[]]byte("balances")` to save some disk space). + BalancesPrefix = []byte{0x02} + SupplyKey = []byte{0x00} + DenomMetadataPrefix = []byte{0x1} +) + +// DenomMetadataKey returns the denomination metadata key. +func DenomMetadataKey(denom string) []byte { + d := []byte(denom) + return append(DenomMetadataPrefix, d...) +} + +// AddressFromBalancesStore returns an account address from a balances prefix +// store. The key must not contain the perfix BalancesPrefix as the prefix store +// iterator discards the actual prefix. +func AddressFromBalancesStore(key []byte) sdk.AccAddress { + addrLen := key[0] + addr := key[1 : addrLen+1] + + return sdk.AccAddress(addr) +} + +// CreateAccountBalancesPrefix creates the prefix for an account's balances. +func CreateAccountBalancesPrefix(addr []byte) []byte { + return append(BalancesPrefix, address.MustLengthPrefix(addr)...) +} diff --git a/x/bank/legacy/v042/store.go b/x/bank/legacy/v042/store.go new file mode 100644 index 000000000000..9295d8d2ff77 --- /dev/null +++ b/x/bank/legacy/v042/store.go @@ -0,0 +1,38 @@ +package v042 + +import ( + "github.com/cosmos/cosmos-sdk/store/prefix" + sdk "github.com/cosmos/cosmos-sdk/types" + v040auth "github.com/cosmos/cosmos-sdk/x/auth/legacy/v040" + v040bank "github.com/cosmos/cosmos-sdk/x/bank/legacy/v040" +) + +// MigrateStore performs in-place store migrations from v0.40 to v0.42. The +// migration includes: +// +// - Change addresses to be length-prefixed. +// - Change balances prefix to 1 byte +func MigrateStore(ctx sdk.Context, storeKey sdk.StoreKey) error { + store := ctx.KVStore(storeKey) + + // old key is of format: + // prefix ("balances") || addrBytes (20 bytes) || denomBytes + // new key is of format + // prefix (0x02) || addrLen (1 byte) || addrBytes || denomBytes + oldStore := prefix.NewStore(store, v040bank.BalancesPrefix) + + oldStoreIter := oldStore.Iterator(nil, nil) + defer oldStoreIter.Close() + + for ; oldStoreIter.Valid(); oldStoreIter.Next() { + addr := v040bank.AddressFromBalancesStore(oldStoreIter.Key()) + denom := oldStoreIter.Key()[v040auth.AddrLen:] + newStoreKey := append(CreateAccountBalancesPrefix(addr), denom...) + + // Set new key on store. Values don't change. + store.Set(newStoreKey, oldStoreIter.Value()) + oldStore.Delete(oldStoreIter.Key()) + } + + return nil +} diff --git a/x/bank/legacy/v042/store_test.go b/x/bank/legacy/v042/store_test.go new file mode 100644 index 000000000000..e1b6a7987a04 --- /dev/null +++ b/x/bank/legacy/v042/store_test.go @@ -0,0 +1,36 @@ +package v042_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/cosmos/cosmos-sdk/testutil" + "github.com/cosmos/cosmos-sdk/testutil/testdata" + sdk "github.com/cosmos/cosmos-sdk/types" + v040bank "github.com/cosmos/cosmos-sdk/x/bank/legacy/v040" + v042bank "github.com/cosmos/cosmos-sdk/x/bank/legacy/v042" +) + +func TestStoreMigration(t *testing.T) { + bankKey := sdk.NewKVStoreKey("bank") + ctx := testutil.DefaultContext(bankKey, sdk.NewTransientStoreKey("transient_test")) + store := ctx.KVStore(bankKey) + + _, _, addr := testdata.KeyTestPubAddr() + denom := []byte("foo") + value := []byte("bar") + + oldKey := append(append(v040bank.BalancesPrefix, addr...), denom...) + store.Set(oldKey, value) + + err := v042bank.MigrateStore(ctx, bankKey) + require.NoError(t, err) + + newKey := append(v042bank.CreateAccountBalancesPrefix(addr), denom...) + // -7 because we replaced "balances" with 0x02, + // +1 because we added length-prefix to address. + require.Equal(t, len(oldKey)-7+1, len(newKey)) + require.Nil(t, store.Get(oldKey)) + require.Equal(t, value, store.Get(newKey)) +} diff --git a/x/bank/module.go b/x/bank/module.go index f271fa21975c..77bff6b73b61 100644 --- a/x/bank/module.go +++ b/x/bank/module.go @@ -100,6 +100,9 @@ type AppModule struct { func (am AppModule) RegisterServices(cfg module.Configurator) { types.RegisterMsgServer(cfg.MsgServer(), keeper.NewMsgServerImpl(am.keeper)) types.RegisterQueryServer(cfg.QueryServer(), am.keeper) + cfg.RegisterMigration(types.ModuleName, 0, func(ctx sdk.Context) error { + return am.keeper.(keeper.MigrationKeeper).Migrate1(ctx) + }) } // NewAppModule creates a new AppModule object @@ -151,6 +154,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 2 } + // BeginBlock performs a no-op. func (AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/bank/spec/01_state.md b/x/bank/spec/01_state.md index f744e2e779a1..6ca6b97e8fc1 100644 --- a/x/bank/spec/01_state.md +++ b/x/bank/spec/01_state.md @@ -7,5 +7,5 @@ order: 1 The `x/bank` module keeps state of two primary objects, account balances and the total supply of all balances. -- Balances: `[]byte("balances") | []byte(address) / []byte(balance.Denom) -> ProtocolBuffer(balance)` - Supply: `0x0 -> ProtocolBuffer(Supply)` +- Balances: `0x2 | byte(address length) | []byte(address) | []byte(balance.Denom) -> ProtocolBuffer(balance)` diff --git a/x/capability/module.go b/x/capability/module.go index 7957f57747d6..06bee1e2d029 100644 --- a/x/capability/module.go +++ b/x/capability/module.go @@ -136,6 +136,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(genState) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock executes all ABCI BeginBlock logic respective to the capability module. func (am AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/crisis/module.go b/x/crisis/module.go index 5d34c1ce28f9..66d829be810e 100644 --- a/x/crisis/module.go +++ b/x/crisis/module.go @@ -158,6 +158,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock performs a no-op. func (AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/distribution/module.go b/x/distribution/module.go index 034be6d9651d..0f0c0d358a91 100644 --- a/x/distribution/module.go +++ b/x/distribution/module.go @@ -161,6 +161,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the distribution module. func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) { BeginBlocker(ctx, req, am.keeper) diff --git a/x/evidence/module.go b/x/evidence/module.go index 4367fe8d58ce..9143e208e716 100644 --- a/x/evidence/module.go +++ b/x/evidence/module.go @@ -175,6 +175,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(ExportGenesis(ctx, am.keeper)) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock executes all ABCI BeginBlock logic respective to the evidence module. func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) { BeginBlocker(ctx, req, am.keeper) diff --git a/x/feegrant/module.go b/x/feegrant/module.go index 5f4ba807d895..31b08e6baab1 100644 --- a/x/feegrant/module.go +++ b/x/feegrant/module.go @@ -167,6 +167,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the feegrant module. func (am AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/genutil/module.go b/x/genutil/module.go index bfaeb0c59168..7a65531e453f 100644 --- a/x/genutil/module.go +++ b/x/genutil/module.go @@ -110,3 +110,6 @@ func (am AppModule) InitGenesis(ctx sdk.Context, cdc codec.JSONMarshaler, data j func (am AppModule) ExportGenesis(_ sdk.Context, cdc codec.JSONMarshaler) json.RawMessage { return am.DefaultGenesis(cdc) } + +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } diff --git a/x/gov/module.go b/x/gov/module.go index ad2191660c08..b72a4bfe5f5c 100644 --- a/x/gov/module.go +++ b/x/gov/module.go @@ -177,6 +177,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock performs a no-op. func (AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/ibc/applications/transfer/module.go b/x/ibc/applications/transfer/module.go index 67c736555b8a..25290d69a64c 100644 --- a/x/ibc/applications/transfer/module.go +++ b/x/ibc/applications/transfer/module.go @@ -145,6 +145,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock implements the AppModule interface func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) { } diff --git a/x/ibc/core/module.go b/x/ibc/core/module.go index 3371dc88a446..6527ab71eb2f 100644 --- a/x/ibc/core/module.go +++ b/x/ibc/core/module.go @@ -156,6 +156,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(ExportGenesis(ctx, *am.keeper)) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the ibc module. func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) { ibcclient.BeginBlocker(ctx, am.keeper.ClientKeeper) diff --git a/x/mint/module.go b/x/mint/module.go index 44e96ce74bda..d33043b36a2e 100644 --- a/x/mint/module.go +++ b/x/mint/module.go @@ -146,6 +146,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the mint module. func (am AppModule) BeginBlock(ctx sdk.Context, _ abci.RequestBeginBlock) { BeginBlocker(ctx, am.keeper) diff --git a/x/params/keeper/common_test.go b/x/params/keeper/common_test.go index f6d567db11f7..3ba444173588 100644 --- a/x/params/keeper/common_test.go +++ b/x/params/keeper/common_test.go @@ -1,14 +1,9 @@ package keeper_test import ( - "github.com/tendermint/tendermint/libs/log" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - dbm "github.com/tendermint/tm-db" - - "github.com/cosmos/cosmos-sdk/simapp" - "github.com/cosmos/cosmos-sdk/codec" - "github.com/cosmos/cosmos-sdk/store" + "github.com/cosmos/cosmos-sdk/simapp" + "github.com/cosmos/cosmos-sdk/testutil" sdk "github.com/cosmos/cosmos-sdk/types" paramskeeper "github.com/cosmos/cosmos-sdk/x/params/keeper" ) @@ -18,7 +13,7 @@ func testComponents() (*codec.LegacyAmino, sdk.Context, sdk.StoreKey, sdk.StoreK legacyAmino := createTestCodec() mkey := sdk.NewKVStoreKey("test") tkey := sdk.NewTransientStoreKey("transient_test") - ctx := defaultContext(mkey, tkey) + ctx := testutil.DefaultContext(mkey, tkey) keeper := paramskeeper.NewKeeper(marshaler, legacyAmino, mkey, tkey) return legacyAmino, ctx, mkey, tkey, keeper @@ -37,16 +32,3 @@ func createTestCodec() *codec.LegacyAmino { cdc.RegisterConcrete(invalid{}, "test/invalid", nil) return cdc } - -func defaultContext(key sdk.StoreKey, tkey sdk.StoreKey) sdk.Context { - db := dbm.NewMemDB() - cms := store.NewCommitMultiStore(db) - cms.MountStoreWithDB(key, sdk.StoreTypeIAVL, db) - cms.MountStoreWithDB(tkey, sdk.StoreTypeTransient, db) - err := cms.LoadLatestVersion() - if err != nil { - panic(err) - } - ctx := sdk.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger()) - return ctx -} diff --git a/x/params/module.go b/x/params/module.go index b0a4584129ef..c1a115b210c1 100644 --- a/x/params/module.go +++ b/x/params/module.go @@ -139,6 +139,9 @@ func (am AppModule) ExportGenesis(_ sdk.Context, _ codec.JSONMarshaler) json.Raw return nil } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock performs a no-op. func (AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/slashing/module.go b/x/slashing/module.go index 91ad472e90df..ba5cfb60adb9 100644 --- a/x/slashing/module.go +++ b/x/slashing/module.go @@ -159,6 +159,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the slashing module. func (am AppModule) BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock) { BeginBlocker(ctx, req, am.keeper) diff --git a/x/staking/module.go b/x/staking/module.go index f2e422117476..48c67e6f5cd1 100644 --- a/x/staking/module.go +++ b/x/staking/module.go @@ -157,6 +157,9 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONMarshaler) json return cdc.MustMarshalJSON(gs) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock returns the begin blocker for the staking module. func (am AppModule) BeginBlock(ctx sdk.Context, _ abci.RequestBeginBlock) { BeginBlocker(ctx, am.keeper) diff --git a/x/upgrade/module.go b/x/upgrade/module.go index 4e4982a324cf..23311c22d5f3 100644 --- a/x/upgrade/module.go +++ b/x/upgrade/module.go @@ -120,6 +120,9 @@ func (am AppModule) ExportGenesis(_ sdk.Context, cdc codec.JSONMarshaler) json.R return am.DefaultGenesis(cdc) } +// ConsensusVersion implements AppModule/ConsensusVersion. +func (AppModule) ConsensusVersion() uint64 { return 1 } + // BeginBlock calls the upgrade module hooks // // CONTRACT: this is registered in BeginBlocker *before* all other modules' BeginBlock functions