refactor(x/genutil,server): add export functions to x/gentutil (#18303)

CHANGELOG.md

@@ -79,6 +79,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### API Breaking Changes
 
+* (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `x/genutil` now handles the application export. `server.AddCommands` does not take an `AppExporter` but instead `genutilcli.Commands` does.
 * (x/gov) [#18173](https://github.com/cosmos/cosmos-sdk/pull/18173) Gov Hooks now returns error and are "blocking" if they fail. Expect for `AfterProposalFailedMinDeposit` and `AfterProposalVotingPeriodEnded` that will log the error and continue.
 * (x/gov/testutil) [#17986](https://github.com/cosmos/cosmos-sdk/pull/18036) `MsgDeposit` has been removed because of AutoCLI migration.
 * (x/staking/testutil) [#17986](https://github.com/cosmos/cosmos-sdk/pull/17986) `MsgRedelegateExec`, `MsgUnbondExec` has been removed because of AutoCLI migration.
@@ -187,6 +188,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### CLI Breaking Changes
 
+* (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `appd export` has moved with other genesis commands, use `appd genesis export` instead.
 * (x/auth/vesting) [#18100](https://github.com/cosmos/cosmos-sdk/pull/18100) `appd tx vesting create-vesting-account` takes an amount of coin as last argument instead of second. Coins are space separated.
 * (x/distribution) [#17963](https://github.com/cosmos/cosmos-sdk/pull/17963) `appd tx distribution withdraw-rewards` now only withdraws rewards for the delegator's own delegations. For withdrawing validators commission, use `appd tx distribution withdraw-validator-commission`.
 

docs/architecture/adr-041-in-place-store-migrations.md

@@ -14,7 +14,7 @@ This ADR introduces a mechanism to perform in-place state store migrations durin
 
 ## Context
 
-When a chain upgrade introduces state-breaking changes inside modules, the current procedure consists of exporting the whole state into a JSON file (via the `simd export` command), running migration scripts on the JSON file (`simd genesis migrate` command), clearing the stores (`simd unsafe-reset-all` command), and starting a new chain with the migrated JSON file as new genesis (optionally with a custom initial block height). An example of such a procedure can be seen [in the Cosmos Hub 3->4 migration guide](https://github.com/cosmos/gaia/blob/v4.0.3/docs/migration/cosmoshub-3.md#upgrade-procedure).
+When a chain upgrade introduces state-breaking changes inside modules, the current procedure consists of exporting the whole state into a JSON file (via the `simd genesis export` command), running migration scripts on the JSON file (`simd genesis migrate` command), clearing the stores (`simd unsafe-reset-all` command), and starting a new chain with the migrated JSON file as new genesis (optionally with a custom initial block height). An example of such a procedure can be seen [in the Cosmos Hub 3->4 migration guide](https://github.com/cosmos/gaia/blob/v4.0.3/docs/migration/cosmoshub-3.md#upgrade-procedure).
 
 This procedure is cumbersome for multiple reasons:
 
@@ -147,15 +147,15 @@ While modules MUST register their migration functions when bumping ConsensusVers
 ### Positive
 
 * Perform chain upgrades without manipulating JSON files.
-* While no benchmark has been made yet, it is probable that in-place store migrations will take less time than JSON migrations. The main reason supporting this claim is that both the `simd export` command on the old binary and the `InitChain` function on the new binary will be skipped.
+* While no benchmark has been made yet, it is probable that in-place store migrations will take less time than JSON migrations. The main reason supporting this claim is that both the `simd genesis export` command on the old binary and the `InitChain` function on the new binary will be skipped.
 
 ### Negative
 
 * Module developers MUST correctly track consensus-breaking changes in their modules. If a consensus-breaking change is introduced in a module without its corresponding `ConsensusVersion()` bump, then the `RunMigrations` function won't detect the migration, and the chain upgrade might be unsuccessful. Documentation should clearly reflect this.
 
 ### Neutral
 
-* The Cosmos SDK will continue to support JSON migrations via the existing `simd export` and `simd genesis migrate` commands.
+* The Cosmos SDK will continue to support JSON migrations via the existing `simd genesis export` and `simd genesis migrate` commands.
 * The current ADR does not allow creating, renaming or deleting stores, only modifying existing store keys and values. The Cosmos SDK already has the `StoreLoader` for those operations.
 
 ## Further Discussions

server/cmt_cmds.go

@@ -372,7 +372,7 @@ func BootstrapStateCmd(appCreator types.AppCreator) *cobra.Command {
  }
  if height == 0 {
  home := serverCtx.Viper.GetString(flags.FlagHome)
- db, err := openDB(home, GetAppDBBackend(serverCtx.Viper))
+ db, err := OpenDB(home, GetAppDBBackend(serverCtx.Viper))
  if err != nil {
  return err
  }

server/constructors_test.go

@@ -8,9 +8,9 @@ import (
  "github.com/stretchr/testify/require"
 )
 
-func Test_openDB(t *testing.T) {
+func Test_OpenDB(t *testing.T) {
  t.Parallel()
- _, err := openDB(t.TempDir(), dbm.GoLevelDBBackend)
+ _, err := OpenDB(t.TempDir(), dbm.GoLevelDBBackend)
  require.NoError(t, err)
 }
 

server/rollback.go

@@ -27,7 +27,7 @@ application.
  RunE: func(cmd *cobra.Command, args []string) error {
  ctx := GetServerContextFromCmd(cmd)
 
- db, err := openDB(ctx.Config.RootDir, GetAppDBBackend(ctx.Viper))
+ db, err := OpenDB(ctx.Config.RootDir, GetAppDBBackend(ctx.Viper))
  if err != nil {
  return err
  }

server/start.go

@@ -27,6 +27,7 @@ import (
  "google.golang.org/grpc"
  "google.golang.org/grpc/credentials/insecure"
 
+ "cosmossdk.io/log"
  pruningtypes "cosmossdk.io/store/pruning/types"
 
  "github.com/cosmos/cosmos-sdk/client"
@@ -113,7 +114,7 @@ func StartCmd(appCreator types.AppCreator) *cobra.Command {
 // CometBFT.
 func StartCmdWithOptions(appCreator types.AppCreator, opts StartCmdOptions) *cobra.Command {
  if opts.DBOpener == nil {
- opts.DBOpener = openDB
+ opts.DBOpener = OpenDB
  }
 
  cmd := &cobra.Command{
@@ -437,7 +438,7 @@ func getAndValidateConfig(svrCtx *Context) (serverconfig.Config, error) {
  return config, nil
 }
 
-// returns a function which returns the genesis doc from the genesis file.
+// getGenDocProvider returns a function which returns the genesis doc from the genesis file.
 func getGenDocProvider(cfg *cmtcfg.Config) func() (*cmttypes.GenesisDoc, error) {
  return func() (*cmttypes.GenesisDoc, error) {
  appGenesis, err := genutiltypes.AppGenesisFromFile(cfg.GenesisFile())
@@ -449,11 +450,11 @@ func getGenDocProvider(cfg *cmtcfg.Config) func() (*cmttypes.GenesisDoc, error)
  }
 }
 
-func setupTraceWriter(svrCtx *Context) (traceWriter io.WriteCloser, cleanup func(), err error) {
+// SetupTraceWriter sets up the trace writer and returns a cleanup function.
+func SetupTraceWriter(logger log.Logger, traceWriterFile string) (traceWriter io.WriteCloser, cleanup func(), err error) {
  // clean up the traceWriter when the server is shutting down
  cleanup = func() {}
 
- traceWriterFile := svrCtx.Viper.GetString(flagTraceStore)
  traceWriter, err = openTraceWriter(traceWriterFile)
  if err != nil {
  return traceWriter, cleanup, err
@@ -463,7 +464,7 @@ func setupTraceWriter(svrCtx *Context) (traceWriter io.WriteCloser, cleanup func
  if traceWriter != nil {
  cleanup = func() {
  if err = traceWriter.Close(); err != nil {
- svrCtx.Logger.Error("failed to close trace writer", "err", err)
+ logger.Error("failed to close trace writer", "err", err)
  }
  }
  }
@@ -626,7 +627,7 @@ func getCtx(svrCtx *Context, block bool) (*errgroup.Group, context.Context) {
 }
 
 func startApp(svrCtx *Context, appCreator types.AppCreator, opts StartCmdOptions) (app types.Application, cleanupFn func(), err error) {
- traceWriter, traceCleanupFn, err := setupTraceWriter(svrCtx)
+ traceWriter, traceCleanupFn, err := SetupTraceWriter(svrCtx.Logger, svrCtx.Viper.GetString(flagTraceStore))
  if err != nil {
  return app, traceCleanupFn, err
  }

server/util.go

@@ -321,7 +321,7 @@ func interceptConfigs(rootViper *viper.Viper, customAppTemplate string, customCo
 }
 
 // add server commands
-func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, appExport types.AppExporter, addStartFlags types.ModuleInitFlags) {
+func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, addStartFlags types.ModuleInitFlags) {
  cometCmd := &cobra.Command{
  Use: "comet",
  Aliases: []string{"cometbft", "tendermint"},
@@ -344,7 +344,6 @@ func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, appExport
  rootCmd.AddCommand(
  startCmd,
  cometCmd,
- ExportCmd(appExport),
  version.NewVersionCommand(),
  NewRollbackCmd(appCreator),
  )
@@ -452,7 +451,8 @@ func addrToIP(addr net.Addr) net.IP {
  return ip
 }
 
-func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
+// OpenDB opens the application database using the appropriate driver.
+func OpenDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
  dataDir := filepath.Join(rootDir, "data")
  return dbm.NewDB("application", backendType, dataDir)
 }

simapp/simd/cmd/commands.go

@@ -51,12 +51,12 @@ func initRootCmd(
  snapshot.Cmd(newApp),
  )
 
- server.AddCommands(rootCmd, newApp, appExport, addModuleInitFlags)
+ server.AddCommands(rootCmd, newApp, addModuleInitFlags)
 
  // add keybase, auxiliary RPC, query, genesis, and tx child commands
  rootCmd.AddCommand(
  server.StatusCommand(),
- genesisCommand(txConfig, basicManager),
+ genesisCommand(txConfig, basicManager, appExport),
  queryCommand(),
  txCommand(),
  keys.Commands(),
@@ -68,8 +68,8 @@ func addModuleInitFlags(startCmd *cobra.Command) {
 }
 
 // genesisCommand builds genesis-related `simd genesis` command. Users may provide application specific commands as a parameter
-func genesisCommand(txConfig client.TxConfig, basicManager module.BasicManager, cmds ...*cobra.Command) *cobra.Command {
- cmd := genutilcli.Commands(txConfig, basicManager)
+func genesisCommand(txConfig client.TxConfig, basicManager module.BasicManager, appExport servertypes.AppExporter, cmds ...*cobra.Command) *cobra.Command {
+ cmd := genutilcli.Commands(txConfig, basicManager, appExport)
 
  for _, subCmd := range cmds {
  cmd.AddCommand(subCmd)

tests/e2e/server/export_test.go → tests/e2e/genutil/export_test.go

@@ -1,7 +1,7 @@
 //go:build e2e
 // +build e2e
 
-package server_test
+package genutil_test
 
 import (
  "bytes"
@@ -27,6 +27,7 @@ import (
  "github.com/cosmos/cosmos-sdk/server/types"
  simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
  "github.com/cosmos/cosmos-sdk/x/genutil"
+ genutilcli "github.com/cosmos/cosmos-sdk/x/genutil/client/cli"
  genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
 )
 
@@ -76,14 +77,14 @@ func TestExportCmd_Height(t *testing.T) {
  {
  "should export correct height with --height",
  []string{
- fmt.Sprintf("--%s=%d", server.FlagHeight, 3),
+ fmt.Sprintf("--height=%d", 3),
  },
  5, 4,
  },
  {
  "should export height 0 with --for-zero-height",
  []string{
- fmt.Sprintf("--%s=%s", server.FlagForZeroHeight, "true"),
+ fmt.Sprintf("--for-zero-height=%s", "true"),
  },
  2, 0,
  },
@@ -196,7 +197,7 @@ func setupApp(t *testing.T, tempDir string) (*simapp.SimApp, context.Context, ge
  _, err = app.Commit()
  assert.NilError(t, err)
 
- cmd := server.ExportCmd(func(_ log.Logger, _ dbm.DB, _ io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string, appOptions types.AppOptions, modulesToExport []string) (types.ExportedApp, error) {
+ cmd := genutilcli.ExportCmd(func(_ log.Logger, _ dbm.DB, _ io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string, appOptions types.AppOptions, modulesToExport []string) (types.ExportedApp, error) {
  var simApp *simapp.SimApp
  if height != -1 {
  simApp = simapp.NewSimApp(logger, db, nil, false, appOptions)

x/genutil/README.md

@@ -12,6 +12,7 @@ The `genutil` package contains a variety of genesis utility functionalities for
 * Genesis file migration
 * CometBFT related initialization
  * Translation of an app genesis to a CometBFT genesis
+* Application state export into a genesis file
 
 ## Genesis
 
@@ -87,3 +88,18 @@ simd genesis validate-genesis
 :::warning
 Validate genesis only validates if the genesis is valid at the **current application binary**. For validating a genesis from a previous version of the application, use the `migrate` command to migrate the genesis to the current version.
 :::
+
+#### export
+
+Export state to genesis file.
+
+```shell
+simd genesis export
+```
+
+Some flags are available to customize the export:
+
+* `--for-zero-height`: export the genesis file for a chain with zero height
+* `--height [height]`: export the genesis file for a chain with a given height
+
+Read the help for more information.

x/genutil/client/cli/commands.go

@@ -6,19 +6,20 @@ import (
  banktypes "cosmossdk.io/x/bank/types"
 
  "github.com/cosmos/cosmos-sdk/client"
+ servertypes "github.com/cosmos/cosmos-sdk/server/types"
  "github.com/cosmos/cosmos-sdk/types/module"
  "github.com/cosmos/cosmos-sdk/x/genutil"
  genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
 )
 
 // Commands adds core sdk's sub-commands into genesis command.
-func Commands(txConfig client.TxConfig, moduleBasics module.BasicManager) *cobra.Command {
- return CommandsWithCustomMigrationMap(txConfig, moduleBasics, MigrationMap)
+func Commands(txConfig client.TxConfig, moduleBasics module.BasicManager, appExport servertypes.AppExporter) *cobra.Command {
+ return CommandsWithCustomMigrationMap(txConfig, moduleBasics, appExport, MigrationMap)
 }
 
 // CommandsWithCustomMigrationMap adds core sdk's sub-commands into genesis command with custom migration map.
 // This custom migration map can be used by the application to add its own migration map.
-func CommandsWithCustomMigrationMap(txConfig client.TxConfig, moduleBasics module.BasicManager, migrationMap genutiltypes.MigrationMap) *cobra.Command {
+func CommandsWithCustomMigrationMap(txConfig client.TxConfig, moduleBasics module.BasicManager, appExport servertypes.AppExporter, migrationMap genutiltypes.MigrationMap) *cobra.Command {
  cmd := &cobra.Command{
  Use: "genesis",
  Short: "Application's genesis-related subcommands",
@@ -34,6 +35,7 @@ func CommandsWithCustomMigrationMap(txConfig client.TxConfig, moduleBasics modul
  CollectGenTxsCmd(banktypes.GenesisBalancesIterator{}, gentxModule.GenTxValidator, txConfig.SigningContext().ValidatorAddressCodec()),
  ValidateGenesisCmd(moduleBasics),
  AddGenesisAccountCmd(txConfig.SigningContext().AddressCodec()),
+ ExportCmd(appExport),
  )
 
  return cmd

server/export.go → x/genutil/client/cli/export.go

@@ -1,4 +1,4 @@
-package server
+package cli
 
 import (
  "bytes"
@@ -10,33 +10,34 @@ import (
  "github.com/spf13/cobra"
 
  "github.com/cosmos/cosmos-sdk/client/flags"
- "github.com/cosmos/cosmos-sdk/server/types"
+ "github.com/cosmos/cosmos-sdk/server"
+ servertypes "github.com/cosmos/cosmos-sdk/server/types"
  "github.com/cosmos/cosmos-sdk/version"
  genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
 )
 
 const (
- FlagHeight = "height"
- FlagForZeroHeight = "for-zero-height"
- FlagJailAllowedAddrs = "jail-allowed-addrs"
- FlagModulesToExport = "modules-to-export"
+ flagTraceStore = "trace-store"
+ flagHeight = "height"
+ flagForZeroHeight = "for-zero-height"
+ flagJailAllowedAddrs = "jail-allowed-addrs"
+ flagModulesToExport = "modules-to-export"
 )
 
 // ExportCmd dumps app state to JSON.
-func ExportCmd(appExporter types.AppExporter) *cobra.Command {
+func ExportCmd(appExporter servertypes.AppExporter) *cobra.Command {
  cmd := &cobra.Command{
  Use: "export",
  Short: "Export state to JSON",
  Args: cobra.NoArgs,
  RunE: func(cmd *cobra.Command, _ []string) error {
- serverCtx := GetServerContextFromCmd(cmd)
- config := serverCtx.Config
+ serverCtx := server.GetServerContextFromCmd(cmd)
 
- if _, err := os.Stat(config.GenesisFile()); os.IsNotExist(err) {
+ if _, err := os.Stat(serverCtx.Config.GenesisFile()); os.IsNotExist(err) {
  return err
  }
 
- db, err := openDB(config.RootDir, GetAppDBBackend(serverCtx.Viper))
+ db, err := server.OpenDB(serverCtx.Config.RootDir, server.GetAppDBBackend(serverCtx.Viper))
  if err != nil {
  return err
  }
@@ -50,7 +51,7 @@ func ExportCmd(appExporter types.AppExporter) *cobra.Command {
  // It is possible that the genesis file is large,
  // so we don't need to read it all into memory
  // before we stream it out.
- f, err := os.OpenFile(config.GenesisFile(), os.O_RDONLY, 0)
+ f, err := os.OpenFile(serverCtx.Config.GenesisFile(), os.O_RDONLY, 0)
  if err != nil {
  return err
  }
@@ -64,15 +65,16 @@ func ExportCmd(appExporter types.AppExporter) *cobra.Command {
  }
 
  traceWriterFile, _ := cmd.Flags().GetString(flagTraceStore)
- traceWriter, err := openTraceWriter(traceWriterFile)
+ traceWriter, cleanup, err := server.SetupTraceWriter(serverCtx.Logger, traceWriterFile)
  if err != nil {
  return err
  }
+ defer cleanup()
 
- height, _ := cmd.Flags().GetInt64(FlagHeight)
- forZeroHeight, _ := cmd.Flags().GetBool(FlagForZeroHeight)
- jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(FlagJailAllowedAddrs)
- modulesToExport, _ := cmd.Flags().GetStringSlice(FlagModulesToExport)
+ height, _ := cmd.Flags().GetInt64(flagHeight)
+ forZeroHeight, _ := cmd.Flags().GetBool(flagForZeroHeight)
+ jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(flagJailAllowedAddrs)
+ modulesToExport, _ := cmd.Flags().GetStringSlice(flagModulesToExport)
  outputDocument, _ := cmd.Flags().GetString(flags.FlagOutputDocument)
 
  exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, modulesToExport)
@@ -112,10 +114,10 @@ func ExportCmd(appExporter types.AppExporter) *cobra.Command {
  },
  }
 
- cmd.Flags().Int64(FlagHeight, -1, "Export state from a particular height (-1 means latest height)")
- cmd.Flags().Bool(FlagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)")
- cmd.Flags().StringSlice(FlagJailAllowedAddrs, []string{}, "Comma-separated list of operator addresses of jailed validators to unjail")
- cmd.Flags().StringSlice(FlagModulesToExport, []string{}, "Comma-separated list of modules to export. If empty, will export all modules")
+ cmd.Flags().Int64(flagHeight, -1, "Export state from a particular height (-1 means latest height)")
+ cmd.Flags().Bool(flagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)")
+ cmd.Flags().StringSlice(flagJailAllowedAddrs, []string{}, "Comma-separated list of operator addresses of jailed validators to unjail")
+ cmd.Flags().StringSlice(flagModulesToExport, []string{}, "Comma-separated list of modules to export. If empty, will export all modules")
  cmd.Flags().String(flags.FlagOutputDocument, "", "Exported state is written to the given file instead of STDOUT")
 
  return cmd