From ebfbd95dc0059da5711dd78a0cf1ea8be2599ef2 Mon Sep 17 00:00:00 2001 From: Federico Kunze <31522760+fedekunze@users.noreply.github.com> Date: Thu, 19 Dec 2019 13:02:59 -0300 Subject: [PATCH] Merge PR #215: Update simulation tests --- app/app.go | 72 +++++--- app/app_test.go | 2 +- app/sim_bench_test.go | 107 ++++++++++++ app/sim_test.go | 375 +++++++++--------------------------------- app/utils.go | 31 ---- go.mod | 2 +- go.sum | 13 +- 7 files changed, 251 insertions(+), 351 deletions(-) create mode 100644 app/sim_bench_test.go delete mode 100644 app/utils.go diff --git a/app/app.go b/app/app.go index baec887dfe..2c14e69e5a 100644 --- a/app/app.go +++ b/app/app.go @@ -86,6 +86,9 @@ func MakeCodec() *codec.Codec { return cdc.Seal() } +// Verify app interface at compile time +var _ simapp.App = (*GaiaApp)(nil) + // GaiaApp extended ABCI application type GaiaApp struct { *bam.BaseApp @@ -97,6 +100,9 @@ type GaiaApp struct { keys map[string]*sdk.KVStoreKey tKeys map[string]*sdk.TransientStoreKey + // subspaces + subspaces map[string]params.Subspace + // keepers accountKeeper auth.AccountKeeper bankKeeper bank.Keeper @@ -142,40 +148,56 @@ func NewGaiaApp( invCheckPeriod: invCheckPeriod, keys: keys, tKeys: tKeys, + subspaces: make(map[string]params.Subspace), } // init params keeper and subspaces app.paramsKeeper = params.NewKeeper(app.cdc, keys[params.StoreKey], tKeys[params.TStoreKey], params.DefaultCodespace) - authSubspace := app.paramsKeeper.Subspace(auth.DefaultParamspace) - bankSubspace := app.paramsKeeper.Subspace(bank.DefaultParamspace) - stakingSubspace := app.paramsKeeper.Subspace(staking.DefaultParamspace) - mintSubspace := app.paramsKeeper.Subspace(mint.DefaultParamspace) - distrSubspace := app.paramsKeeper.Subspace(distr.DefaultParamspace) - slashingSubspace := app.paramsKeeper.Subspace(slashing.DefaultParamspace) - govSubspace := app.paramsKeeper.Subspace(gov.DefaultParamspace).WithKeyTable(gov.ParamKeyTable()) - crisisSubspace := app.paramsKeeper.Subspace(crisis.DefaultParamspace) - evidenceSubspace := app.paramsKeeper.Subspace(evidence.DefaultParamspace) + app.subspaces[auth.ModuleName] = app.paramsKeeper.Subspace(auth.DefaultParamspace) + app.subspaces[bank.ModuleName] = app.paramsKeeper.Subspace(bank.DefaultParamspace) + app.subspaces[staking.ModuleName] = app.paramsKeeper.Subspace(staking.DefaultParamspace) + app.subspaces[mint.ModuleName] = app.paramsKeeper.Subspace(mint.DefaultParamspace) + app.subspaces[distr.ModuleName] = app.paramsKeeper.Subspace(distr.DefaultParamspace) + app.subspaces[slashing.ModuleName] = app.paramsKeeper.Subspace(slashing.DefaultParamspace) + app.subspaces[gov.ModuleName] = app.paramsKeeper.Subspace(gov.DefaultParamspace).WithKeyTable(gov.ParamKeyTable()) + app.subspaces[crisis.ModuleName] = app.paramsKeeper.Subspace(crisis.DefaultParamspace) + app.subspaces[evidence.ModuleName] = app.paramsKeeper.Subspace(evidence.DefaultParamspace) // add keepers - app.accountKeeper = auth.NewAccountKeeper(app.cdc, keys[auth.StoreKey], authSubspace, auth.ProtoBaseAccount) - app.bankKeeper = bank.NewBaseKeeper(app.accountKeeper, bankSubspace, bank.DefaultCodespace, app.ModuleAccountAddrs()) - app.supplyKeeper = supply.NewKeeper(app.cdc, keys[supply.StoreKey], app.accountKeeper, app.bankKeeper, maccPerms) + app.accountKeeper = auth.NewAccountKeeper( + app.cdc, keys[auth.StoreKey], app.subspaces[auth.ModuleName], auth.ProtoBaseAccount, + ) + app.bankKeeper = bank.NewBaseKeeper( + app.accountKeeper, app.subspaces[bank.ModuleName], bank.DefaultCodespace, + app.ModuleAccountAddrs(), + ) + app.supplyKeeper = supply.NewKeeper( + app.cdc, keys[supply.StoreKey], app.accountKeeper, app.bankKeeper, maccPerms, + ) stakingKeeper := staking.NewKeeper( - app.cdc, keys[staking.StoreKey], app.supplyKeeper, stakingSubspace, staking.DefaultCodespace, + app.cdc, keys[staking.StoreKey], app.supplyKeeper, + app.subspaces[staking.ModuleName], staking.DefaultCodespace, + ) + app.mintKeeper = mint.NewKeeper( + app.cdc, keys[mint.StoreKey], app.subspaces[mint.ModuleName], &stakingKeeper, + app.supplyKeeper, auth.FeeCollectorName, + ) + app.distrKeeper = distr.NewKeeper( + app.cdc, keys[distr.StoreKey], app.subspaces[distr.ModuleName], &stakingKeeper, + app.supplyKeeper, distr.DefaultCodespace, auth.FeeCollectorName, app.ModuleAccountAddrs(), ) - app.mintKeeper = mint.NewKeeper(app.cdc, keys[mint.StoreKey], mintSubspace, &stakingKeeper, app.supplyKeeper, auth.FeeCollectorName) - app.distrKeeper = distr.NewKeeper(app.cdc, keys[distr.StoreKey], distrSubspace, &stakingKeeper, - app.supplyKeeper, distr.DefaultCodespace, auth.FeeCollectorName, app.ModuleAccountAddrs()) app.slashingKeeper = slashing.NewKeeper( - app.cdc, keys[slashing.StoreKey], &stakingKeeper, slashingSubspace, slashing.DefaultCodespace, + app.cdc, keys[slashing.StoreKey], &stakingKeeper, app.subspaces[slashing.ModuleName], slashing.DefaultCodespace, + ) + app.crisisKeeper = crisis.NewKeeper( + app.subspaces[crisis.ModuleName], invCheckPeriod, app.supplyKeeper, auth.FeeCollectorName, ) - app.crisisKeeper = crisis.NewKeeper(crisisSubspace, invCheckPeriod, app.supplyKeeper, auth.FeeCollectorName) app.upgradeKeeper = upgrade.NewKeeper(keys[upgrade.StoreKey], app.cdc) // create evidence keeper with evidence router evidenceKeeper := evidence.NewKeeper( - app.cdc, keys[evidence.StoreKey], evidenceSubspace, evidence.DefaultCodespace, - &stakingKeeper, app.slashingKeeper, + app.cdc, keys[evidence.StoreKey], app.subspaces[evidence.ModuleName], + evidence.DefaultCodespace, &stakingKeeper, app.slashingKeeper, ) evidenceRouter := evidence.NewRouter() @@ -191,7 +213,7 @@ func NewGaiaApp( AddRoute(distr.RouterKey, distr.NewCommunityPoolSpendProposalHandler(app.distrKeeper)). AddRoute(upgrade.RouterKey, upgrade.NewSoftwareUpgradeProposalHandler(app.upgradeKeeper)) app.govKeeper = gov.NewKeeper( - app.cdc, keys[gov.StoreKey], govSubspace, + app.cdc, keys[gov.StoreKey], app.subspaces[gov.ModuleName], app.supplyKeeper, &stakingKeeper, gov.DefaultCodespace, govRouter, ) @@ -272,6 +294,9 @@ func NewGaiaApp( return app } +// Name returns the name of the App +func (app *GaiaApp) Name() string { return app.BaseApp.Name() } + // BeginBlocker application updates every begin block func (app *GaiaApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.ResponseBeginBlock { return app.mm.BeginBlock(ctx, req) @@ -310,6 +335,11 @@ func (app *GaiaApp) Codec() *codec.Codec { return app.cdc } +// SimulationManager implements the SimulationApp interface +func (app *GaiaApp) SimulationManager() *module.SimulationManager { + return app.sm +} + // GetMaccPerms returns a mapping of the application's module account permissions. func GetMaccPerms() map[string][]string { modAccPerms := make(map[string][]string) diff --git a/app/app_test.go b/app/app_test.go index 1052573b3b..3516fdb865 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -38,7 +38,7 @@ func TestBlackListedAddrs(t *testing.T) { func setGenesis(gapp *GaiaApp) error { genesisState := simapp.NewDefaultGenesisState() - stateBytes, err := codec.MarshalJSONIndent(gapp.cdc, genesisState) + stateBytes, err := codec.MarshalJSONIndent(gapp.Codec(), genesisState) if err != nil { return err } diff --git a/app/sim_bench_test.go b/app/sim_bench_test.go new file mode 100644 index 0000000000..97affcf24a --- /dev/null +++ b/app/sim_bench_test.go @@ -0,0 +1,107 @@ +package app + +import ( + "fmt" + "os" + "testing" + + "github.com/cosmos/cosmos-sdk/simapp" + "github.com/cosmos/cosmos-sdk/x/simulation" + abci "github.com/tendermint/tendermint/abci/types" +) + +// Profile with: +// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/simapp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out +func BenchmarkFullAppSimulation(b *testing.B) { + config, db, dir, logger, _, err := simapp.SetupSimulation("goleveldb-app-sim", "Simulation") + if err != nil { + b.Fatalf("simulation setup failed: %s", err.Error()) + } + + defer func() { + db.Close() + err = os.RemoveAll(dir) + if err != nil { + b.Fatal(err) + } + }() + + app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) + + // run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + b, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(app, app.Codec(), config), + app.ModuleAccountAddrs(), config, + ) + + // export state and simParams before the simulation error is checked + if err = simapp.CheckExportSimulation(app, config, simParams); err != nil { + b.Fatal(err) + } + + if simErr != nil { + b.Fatal(simErr) + } + + if config.Commit { + simapp.PrintStats(db) + } +} + +func BenchmarkInvariants(b *testing.B) { + config, db, dir, logger, _, err := simapp.SetupSimulation("leveldb-app-invariant-bench", "Simulation") + if err != nil { + b.Fatalf("simulation setup failed: %s", err.Error()) + } + + config.AllInvariants = false + + defer func() { + db.Close() + err = os.RemoveAll(dir) + if err != nil { + b.Fatal(err) + } + }() + + app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) + + // run randomized simulation + _, simParams, simErr := simulation.SimulateFromSeed( + b, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(app, app.Codec(), config), + app.ModuleAccountAddrs(), config, + ) + + // export state and simParams before the simulation error is checked + if err = simapp.CheckExportSimulation(app, config, simParams); err != nil { + b.Fatal(err) + } + + if simErr != nil { + b.Fatal(simErr) + } + + if config.Commit { + simapp.PrintStats(db) + } + + ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1}) + + // 3. Benchmark each invariant separately + // + // NOTE: We use the crisis keeper as it has all the invariants registered with + // their respective metadata which makes it useful for testing/benchmarking. + for _, cr := range app.crisisKeeper.Routes() { + cr := cr + b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) { + if res, stop := cr.Invar(ctx); stop { + b.Fatalf( + "broken invariant at block %d of %d\n%s", + ctx.BlockHeight()-1, config.NumBlocks, res, + ) + } + }) + } +} diff --git a/app/sim_test.go b/app/sim_test.go index 5a7d8deffd..ed32360328 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -3,7 +3,6 @@ package app import ( "encoding/json" "fmt" - "io/ioutil" "math/rand" "os" "testing" @@ -15,6 +14,7 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/simapp" + "github.com/cosmos/cosmos-sdk/simapp/helpers" "github.com/cosmos/cosmos-sdk/store" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" @@ -32,6 +32,12 @@ func init() { simapp.GetSimulatorFlags() } +type StoreKeysPrefixes struct { + A sdk.StoreKey + B sdk.StoreKey + Prefixes [][]byte +} + // fauxMerkleModeOpt returns a BaseApp option to use a dbStoreAdapter instead of // an IAVLStore for faster simulation speed. func fauxMerkleModeOpt(bapp *baseapp.BaseApp) { @@ -44,212 +50,96 @@ func interBlockCacheOpt() func(*baseapp.BaseApp) { return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager()) } -// Profile with: -// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/GaiaApp -bench ^BenchmarkFullAppSimulation$ -Commit=true -cpuprofile cpu.out -func BenchmarkFullAppSimulation(b *testing.B) { - logger := log.NewNopLogger() - config := simapp.NewConfigFromFlags() - - var db dbm.DB - dir, err := ioutil.TempDir("", "goleveldb-app-sim") - if err != nil { - fmt.Println(err) - b.Fail() - } - db, err = sdk.NewLevelDB("Simulation", dir) - if err != nil { - fmt.Println(err) - b.Fail() - } - defer func() { - db.Close() - _ = os.RemoveAll(dir) - }() - - app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) - - // Run randomized simulation - // TODO: parameterize numbers, save for a later PR - _, simParams, simErr := simulation.SimulateFromSeed( - b, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), - app.ModuleAccountAddrs(), config, - ) - - // export state and params before the simulation error is checked - if config.ExportStatePath != "" { - if err := exportStateToJSON(app, config.ExportStatePath); err != nil { - fmt.Println(err) - b.Fail() - } - } - - if config.ExportParamsPath != "" { - if err := simapp.ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil { - fmt.Println(err) - b.Fail() - } - } - - if simErr != nil { - fmt.Println(simErr) - b.FailNow() - } - - if config.Commit { - fmt.Println("\nGoLevelDB Stats") - fmt.Println(db.Stats()["leveldb.stats"]) - fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) - } -} - func TestFullAppSimulation(t *testing.T) { - if !simapp.FlagEnabledValue { + config, db, dir, logger, skip, err := simapp.SetupSimulation("leveldb-app-sim", "Simulation") + if skip { t.Skip("skipping application simulation") } - - var logger log.Logger - config := simapp.NewConfigFromFlags() - - if simapp.FlagVerboseValue { - logger = log.TestingLogger() - } else { - logger = log.NewNopLogger() - } - - var db dbm.DB - dir, err := ioutil.TempDir("", "goleveldb-app-sim") - require.NoError(t, err) - db, err = sdk.NewLevelDB("Simulation", dir) - require.NoError(t, err) + require.NoError(t, err, "simulation setup failed") defer func() { db.Close() - _ = os.RemoveAll(dir) + require.NoError(t, os.RemoveAll(dir)) }() app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, fauxMerkleModeOpt) - require.Equal(t, "GaiaApp", app.Name()) + require.Equal(t, appName, app.Name()) - // Run randomized simulation + // run randomized simulation _, simParams, simErr := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), + t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) - // export state and params before the simulation error is checked - if config.ExportStatePath != "" { - err := exportStateToJSON(app, config.ExportStatePath) - require.NoError(t, err) - } - - if config.ExportParamsPath != "" { - err := simapp.ExportParamsToJSON(simParams, config.ExportParamsPath) - require.NoError(t, err) - } - + // export state and simParams before the simulation error is checked + err = simapp.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) require.NoError(t, simErr) if config.Commit { - // for memdb: - // fmt.Println("Database Size", db.Stats()["database.size"]) - fmt.Println("\nGoLevelDB Stats") - fmt.Println(db.Stats()["leveldb.stats"]) - fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) + simapp.PrintStats(db) } } func TestAppImportExport(t *testing.T) { - if !simapp.FlagEnabledValue { + config, db, dir, logger, skip, err := simapp.SetupSimulation("leveldb-app-sim", "Simulation") + if skip { t.Skip("skipping application import/export simulation") } - - var logger log.Logger - config := simapp.NewConfigFromFlags() - - if simapp.FlagVerboseValue { - logger = log.TestingLogger() - } else { - logger = log.NewNopLogger() - } - - var db dbm.DB - dir, err := ioutil.TempDir("", "goleveldb-app-sim") - require.NoError(t, err) - db, err = sdk.NewLevelDB("Simulation", dir) - require.NoError(t, err) + require.NoError(t, err, "simulation setup failed") defer func() { db.Close() - _ = os.RemoveAll(dir) + require.NoError(t, os.RemoveAll(dir)) }() app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, fauxMerkleModeOpt) - require.Equal(t, "SimApp", app.Name()) + require.Equal(t, appName, app.Name()) // Run randomized simulation _, simParams, simErr := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), + t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) // export state and simParams before the simulation error is checked - if config.ExportStatePath != "" { - err := exportStateToJSON(app, config.ExportStatePath) - require.NoError(t, err) - } - - if config.ExportParamsPath != "" { - err := simapp.ExportParamsToJSON(simParams, config.ExportParamsPath) - require.NoError(t, err) - } - + err = simapp.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) require.NoError(t, simErr) if config.Commit { - // for memdb: - // fmt.Println("Database Size", db.Stats()["database.size"]) - fmt.Println("\nGoLevelDB Stats") - fmt.Println(db.Stats()["leveldb.stats"]) - fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) + simapp.PrintStats(db) } fmt.Printf("exporting genesis...\n") appState, _, err := app.ExportAppStateAndValidators(false, []string{}) require.NoError(t, err) + fmt.Printf("importing genesis...\n") - newDir, err := ioutil.TempDir("", "goleveldb-app-sim-2") - require.NoError(t, err) - newDB, err := sdk.NewLevelDB("Simulation-2", dir) - require.NoError(t, err) + _, newDB, newDir, _, _, err := simapp.SetupSimulation("leveldb-app-sim-2", "Simulation-2") + require.NoError(t, err, "simulation setup failed") defer func() { newDB.Close() - _ = os.RemoveAll(newDir) + require.NoError(t, os.RemoveAll(newDir)) }() newApp := NewGaiaApp(log.NewNopLogger(), newDB, nil, true, simapp.FlagPeriodValue, fauxMerkleModeOpt) - require.Equal(t, "SimApp", newApp.Name()) + require.Equal(t, appName, newApp.Name()) - var genesisState simapp.GenesisState - err = app.cdc.UnmarshalJSON(appState, &genesisState) + var genesisState GenesisState + err = app.Codec().UnmarshalJSON(appState, &genesisState) require.NoError(t, err) + ctxA := app.NewContext(true, abci.Header{Height: app.LastBlockHeight()}) ctxB := newApp.NewContext(true, abci.Header{Height: app.LastBlockHeight()}) newApp.mm.InitGenesis(ctxB, genesisState) fmt.Printf("comparing stores...\n") - ctxA := app.NewContext(true, abci.Header{Height: app.LastBlockHeight()}) - - type StoreKeysPrefixes struct { - A sdk.StoreKey - B sdk.StoreKey - Prefixes [][]byte - } storeKeysPrefixes := []StoreKeysPrefixes{ {app.keys[baseapp.MainStoreKey], newApp.keys[baseapp.MainStoreKey], [][]byte{}}, @@ -266,122 +156,84 @@ func TestAppImportExport(t *testing.T) { {app.keys[gov.StoreKey], newApp.keys[gov.StoreKey], [][]byte{}}, } - for _, storeKeysPrefix := range storeKeysPrefixes { - storeKeyA := storeKeysPrefix.A - storeKeyB := storeKeysPrefix.B - prefixes := storeKeysPrefix.Prefixes - - storeA := ctxA.KVStore(storeKeyA) - storeB := ctxB.KVStore(storeKeyB) + for _, skp := range storeKeysPrefixes { + storeA := ctxA.KVStore(skp.A) + storeB := ctxB.KVStore(skp.B) - failedKVAs, failedKVBs := sdk.DiffKVStores(storeA, storeB, prefixes) + failedKVAs, failedKVBs := sdk.DiffKVStores(storeA, storeB, skp.Prefixes) require.Equal(t, len(failedKVAs), len(failedKVBs), "unequal sets of key-values to compare") - fmt.Printf("compared %d key/value pairs between %s and %s\n", len(failedKVAs), storeKeyA, storeKeyB) - require.Len(t, failedKVAs, 0, simapp.GetSimulationLog(storeKeyA.Name(), app.sm.StoreDecoders, app.cdc, failedKVAs, failedKVBs)) + fmt.Printf("compared %d key/value pairs between %s and %s\n", len(failedKVAs), skp.A, skp.B) + require.Equal(t, len(failedKVAs), 0, simapp.GetSimulationLog(skp.A.Name(), app.SimulationManager().StoreDecoders, app.Codec(), failedKVAs, failedKVBs)) } } func TestAppSimulationAfterImport(t *testing.T) { - if !simapp.FlagEnabledValue { + config, db, dir, logger, skip, err := simapp.SetupSimulation("leveldb-app-sim", "Simulation") + if skip { t.Skip("skipping application simulation after import") } - - var logger log.Logger - config := simapp.NewConfigFromFlags() - - if simapp.FlagVerboseValue { - logger = log.TestingLogger() - } else { - logger = log.NewNopLogger() - } - - dir, err := ioutil.TempDir("", "goleveldb-app-sim") - require.NoError(t, err) - db, err := sdk.NewLevelDB("Simulation", dir) - require.NoError(t, err) + require.NoError(t, err, "simulation setup failed") defer func() { db.Close() - _ = os.RemoveAll(dir) + require.NoError(t, os.RemoveAll(dir)) }() app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, fauxMerkleModeOpt) - require.Equal(t, "GaiaApp", app.Name()) + require.Equal(t, appName, app.Name()) - // Run randomized simulation // Run randomized simulation stopEarly, simParams, simErr := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), + t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) - // export state and params before the simulation error is checked - if config.ExportStatePath != "" { - err := exportStateToJSON(app, config.ExportStatePath) - require.NoError(t, err) - } - - if config.ExportParamsPath != "" { - err := simapp.ExportParamsToJSON(simParams, config.ExportParamsPath) - require.NoError(t, err) - } - + // export state and simParams before the simulation error is checked + err = simapp.CheckExportSimulation(app, config, simParams) + require.NoError(t, err) require.NoError(t, simErr) if config.Commit { - // for memdb: - // fmt.Println("Database Size", db.Stats()["database.size"]) - fmt.Println("\nGoLevelDB Stats") - fmt.Println(db.Stats()["leveldb.stats"]) - fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"]) + simapp.PrintStats(db) } if stopEarly { - // we can't export or import a zero-validator genesis - fmt.Printf("We can't export or import a zero-validator genesis, exiting test...\n") + fmt.Println("can't export or import a zero-validator genesis, exiting test...") return } - fmt.Printf("Exporting genesis...\n") + fmt.Printf("exporting genesis...\n") appState, _, err := app.ExportAppStateAndValidators(true, []string{}) - if err != nil { - panic(err) - } + require.NoError(t, err) - fmt.Printf("Importing genesis...\n") + fmt.Printf("importing genesis...\n") - newDir, err := ioutil.TempDir("", "goleveldb-app-sim-2") - require.NoError(t, err) - newDB, err := sdk.NewLevelDB("Simulation-2", dir) - require.NoError(t, err) + _, newDB, newDir, _, _, err := simapp.SetupSimulation("leveldb-app-sim-2", "Simulation-2") + require.NoError(t, err, "simulation setup failed") defer func() { newDB.Close() - _ = os.RemoveAll(newDir) + require.NoError(t, os.RemoveAll(newDir)) }() - newApp := NewGaiaApp(log.NewNopLogger(), newDB, nil, true, 0, fauxMerkleModeOpt) - require.Equal(t, "GaiaApp", newApp.Name()) + newApp := NewGaiaApp(log.NewNopLogger(), newDB, nil, true, simapp.FlagPeriodValue, fauxMerkleModeOpt) + require.Equal(t, appName, newApp.Name()) newApp.InitChain(abci.RequestInitChain{ AppStateBytes: appState, }) - // Run randomized simulation on imported app _, _, err = simulation.SimulateFromSeed( - t, os.Stdout, newApp.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), + t, os.Stdout, newApp.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(newApp, newApp.Codec(), config), newApp.ModuleAccountAddrs(), config, ) - require.NoError(t, err) } -// TODO: Make another test for the fuzzer itself, which just has noOp txs -// and doesn't depend on the application. func TestAppStateDeterminism(t *testing.T) { if !simapp.FlagEnabledValue { t.Skip("skipping application simulation") @@ -392,6 +244,7 @@ func TestAppStateDeterminism(t *testing.T) { config.ExportParamsPath = "" config.OnOperation = false config.AllInvariants = false + config.ChainID = helpers.SimAppChainID numSeeds := 3 numTimesToRunPerSeed := 5 @@ -401,8 +254,15 @@ func TestAppStateDeterminism(t *testing.T) { config.Seed = rand.Int63() for j := 0; j < numTimesToRunPerSeed; j++ { - logger := log.NewNopLogger() + var logger log.Logger + if simapp.FlagVerboseValue { + logger = log.TestingLogger() + } else { + logger = log.NewNopLogger() + } + db := dbm.NewMemDB() + app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) fmt.Printf( @@ -411,12 +271,16 @@ func TestAppStateDeterminism(t *testing.T) { ) _, _, err := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), + t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.SimulationManager()), + simapp.SimulationOperations(app, app.Codec(), config), app.ModuleAccountAddrs(), config, ) require.NoError(t, err) + if config.Commit { + simapp.PrintStats(db) + } + appHash := app.LastCommitID().Hash appHashList[j] = appHash @@ -429,82 +293,3 @@ func TestAppStateDeterminism(t *testing.T) { } } } - -func BenchmarkInvariants(b *testing.B) { - logger := log.NewNopLogger() - - config := simapp.NewConfigFromFlags() - config.AllInvariants = false - - dir, err := ioutil.TempDir("", "goleveldb-app-invariant-bench") - if err != nil { - fmt.Println(err) - b.Fail() - } - db, err := sdk.NewLevelDB("simulation", dir) - if err != nil { - fmt.Println(err) - b.Fail() - } - - defer func() { - db.Close() - os.RemoveAll(dir) - }() - - app := NewGaiaApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) - - // 2. Run parameterized simulation (w/o invariants) - _, simParams, simErr := simulation.SimulateFromSeed( - b, ioutil.Discard, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - SimulationOperations(app, app.Codec(), config), - app.ModuleAccountAddrs(), config, - ) - - // export state and params before the simulation error is checked - if config.ExportStatePath != "" { - if err := exportStateToJSON(app, config.ExportStatePath); err != nil { - fmt.Println(err) - b.Fail() - } - } - - if config.ExportParamsPath != "" { - if err := simapp.ExportParamsToJSON(simParams, config.ExportParamsPath); err != nil { - fmt.Println(err) - b.Fail() - } - } - - if simErr != nil { - fmt.Println(simErr) - b.FailNow() - } - - ctx := app.NewContext(true, abci.Header{Height: app.LastBlockHeight() + 1}) - - // 3. Benchmark each invariant separately - // - // NOTE: We use the crisis keeper as it has all the invariants registered with - // their respective metadata which makes it useful for testing/benchmarking. - for _, cr := range app.crisisKeeper.Routes() { - cr := cr - b.Run(fmt.Sprintf("%s/%s", cr.ModuleName, cr.Route), func(b *testing.B) { - if res, stop := cr.Invar(ctx); stop { - fmt.Printf("broken invariant at block %d of %d\n%s", ctx.BlockHeight()-1, config.NumBlocks, res) - b.FailNow() - } - }) - } -} - -// auxiliary function to export the app state to JSON -func exportStateToJSON(app *GaiaApp, path string) error { - fmt.Println("exporting app state...") - appState, _, err := app.ExportAppStateAndValidators(false, nil) - if err != nil { - return err - } - - return ioutil.WriteFile(path, []byte(appState), 0644) -} diff --git a/app/utils.go b/app/utils.go deleted file mode 100644 index f749b5286a..0000000000 --- a/app/utils.go +++ /dev/null @@ -1,31 +0,0 @@ -package app - -import ( - "io/ioutil" - - "github.com/cosmos/cosmos-sdk/codec" - "github.com/cosmos/cosmos-sdk/types/module" - "github.com/cosmos/cosmos-sdk/x/simulation" -) - -// SimulationOperations retrieves the simulation params from the provided file path -// and returns all the modules weighted operations -func SimulationOperations(app *GaiaApp, cdc *codec.Codec, config simulation.Config) []simulation.WeightedOperation { - simState := module.SimulationState{ - AppParams: make(simulation.AppParams), - Cdc: cdc, - } - - if config.ParamsFile != "" { - bz, err := ioutil.ReadFile(config.ParamsFile) - if err != nil { - panic(err) - } - - app.cdc.MustUnmarshalJSON(bz, &simState.AppParams) - } - - simState.ParamChanges = app.sm.GenerateParamChanges(config.Seed) - simState.Contents = app.sm.GetProposalContents(simState) - return app.sm.WeightedOperations(simState) -} diff --git a/go.mod b/go.mod index 83494e2137..36d8a6a594 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/btcsuite/btcd v0.0.0-20190807005414-4063feeff79a // indirect - github.com/cosmos/cosmos-sdk v0.34.4-0.20191213112149-d7b0f4b9b4fb + github.com/cosmos/cosmos-sdk v0.34.4-0.20191218181537-cf4645213d41 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d // indirect github.com/golang/mock v1.3.1 // indirect github.com/onsi/ginkgo v1.8.0 // indirect diff --git a/go.sum b/go.sum index ba93560f8a..e2e6ce0f61 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,15 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosmos/cosmos-sdk v0.34.4-0.20191213112149-d7b0f4b9b4fb h1:zVivJCmI6SF3DmxlhY94trezOyfPXtiIDxCH3VPFXHY= -github.com/cosmos/cosmos-sdk v0.34.4-0.20191213112149-d7b0f4b9b4fb/go.mod h1:hasIdlU9b3FEFCWpoStvNQQPg1ZpAKnpmlFklAk1W1o= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191205150729-0300a6f6d7a7 h1:W2WVv1zwI34A0XsPdUXf+k1etTSftA5cNHXizaJ4xbw= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191205150729-0300a6f6d7a7/go.mod h1:JWuSAxZmMgNmNsZBCTuFfMHeeAAJZDxTnAKrQeSJOdk= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191209121439-88f1e4eca5cf h1:VDXm4Y3qVMNF9icZjDo6CBICDrxf9UW0U7L0MEaSPdA= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191209121439-88f1e4eca5cf/go.mod h1:Bzqy/gA/MZ4fkU/ia6ddMso3FEatgAsKBX9yI4QwDGI= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191209200230-8dd95bd01427 h1:XPi4aJ1c587WKeEaMx5Dz7UB5qAH++VPEN3aNIzvrfc= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191209200230-8dd95bd01427/go.mod h1:Bzqy/gA/MZ4fkU/ia6ddMso3FEatgAsKBX9yI4QwDGI= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191218181537-cf4645213d41 h1:QVFH8qAihoIDUdaeBpge+mdS5QfBrwP0rpi0fp54IeI= +github.com/cosmos/cosmos-sdk v0.34.4-0.20191218181537-cf4645213d41/go.mod h1:hasIdlU9b3FEFCWpoStvNQQPg1ZpAKnpmlFklAk1W1o= +github.com/cosmos/cosmos-sdk v0.37.4 h1:1ioXxkpiS+wOgaUbROeDIyuF7hciU5nti0TSyBmV2Ok= github.com/cosmos/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= @@ -163,6 +170,7 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -347,6 +355,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=