Skip to content

Commit

Permalink
feat: Add a cli cmd to prune old states according to current settings…
Browse files Browse the repository at this point in the history
… (#12742)

* add PruningCmd and change PruneStores signature

* the mimimum default pruning interval is 10

Co-authored-by: Marko <marbar3778@yahoo.com>
  • Loading branch information
adu-crypto and tac0turtle authored Sep 1, 2022
1 parent d638ca3 commit d874ace
Show file tree
Hide file tree
Showing 4 changed files with 142 additions and 11 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
* (testutil) [#12973](https://github.com/cosmos/cosmos-sdk/pull/12973) Add generic `testutil.RandSliceElem` function which selects a random element from the list.
* (client) [#12936](https://github.com/cosmos/cosmos-sdk/pull/12936) Add capability to preprocess transactions before broadcasting from a higher level chain.
* (x/authz) [#13047](https://github.com/cosmos/cosmos-sdk/pull/13047) Add a GetAuthorization function to the keeper.
* (cli) [#12742](https://github.com/cosmos/cosmos-sdk/pull/12742) Add the `prune` CLI cmd to manually prune app store history versions based on the pruning options.

### Improvements

Expand Down
117 changes: 117 additions & 0 deletions client/pruning/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package pruning

import (
"fmt"
"os"
"path/filepath"

"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/cosmos/cosmos-sdk/client/flags"
pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types"
"github.com/cosmos/cosmos-sdk/server"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/store/rootmulti"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
)

const FlagAppDBBackend = "app-db-backend"

// PruningCmd prunes the sdk root multi store history versions based on the pruning options
// specified by command flags.
func PruningCmd(appCreator servertypes.AppCreator) *cobra.Command {
cmd := &cobra.Command{
Use: "prune",
Short: "Prune app history states by keeping the recent heights and deleting old heights",
Long: `Prune app history states by keeping the recent heights and deleting old heights.
The pruning option is provided via the '--pruning' flag or alternatively with '--pruning-keep-recent'
For '--pruning' the options are as follows:
default: the last 362880 states are kept
nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)
everything: 2 latest states will be kept
custom: allow pruning options to be manually specified through 'pruning-keep-recent'.
besides pruning options, database home directory and database backend type should also be specified via flags
'--home' and '--app-db-backend'.
valid app-db-backend type includes 'goleveldb', 'cleveldb', 'rocksdb', 'boltdb', and 'badgerdb'.
`,
Example: "prune --home './' --app-db-backend 'goleveldb' --pruning 'custom' --pruning-keep-recent 100",
RunE: func(cmd *cobra.Command, _ []string) error {
vp := viper.New()

// Bind flags to the Context's Viper so we can get pruning options.
if err := vp.BindPFlags(cmd.Flags()); err != nil {
return err
}
pruningOptions, err := server.GetPruningOptionsFromFlags(vp)
if err != nil {
return err
}
fmt.Printf("get pruning options from command flags, strategy: %v, keep-recent: %v\n",
pruningOptions.Strategy,
pruningOptions.KeepRecent,
)

home := vp.GetString(flags.FlagHome)
db, err := openDB(home, server.GetAppDBBackend(vp))
if err != nil {
return err
}

logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
app := appCreator(logger, db, nil, vp)
cms := app.CommitMultiStore()

rootMultiStore, ok := cms.(*rootmulti.Store)
if !ok {
return fmt.Errorf("currently only support the pruning of rootmulti.Store type")
}
latestHeight := rootmulti.GetLatestVersion(db)
// valid heights should be greater than 0.
if latestHeight <= 0 {
return fmt.Errorf("the database has no valid heights to prune, the latest height: %v", latestHeight)
}

var pruningHeights []int64
for height := int64(1); height < latestHeight; height++ {
if height < latestHeight-int64(pruningOptions.KeepRecent) {
pruningHeights = append(pruningHeights, height)
}
}
if len(pruningHeights) == 0 {
fmt.Printf("no heights to prune\n")
return nil
}
fmt.Printf(
"pruning heights start from %v, end at %v\n",
pruningHeights[0],
pruningHeights[len(pruningHeights)-1],
)

err = rootMultiStore.PruneStores(false, pruningHeights)
if err != nil {
return err
}
fmt.Printf("successfully pruned the application root multi stores\n")
return nil
},
}

cmd.Flags().String(flags.FlagHome, "", "The database home directory")
cmd.Flags().String(FlagAppDBBackend, "", "The type of database for application and snapshots databases")
cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)")
cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')")
cmd.Flags().Uint64(server.FlagPruningInterval, 10,
`Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom'),
this is not used by this command but kept for compatibility with the complete pruning options`)

return cmd
}

func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
dataDir := filepath.Join(rootDir, "data")
return dbm.NewDB("application", backendType, dataDir)
}
2 changes: 2 additions & 0 deletions simapp/simd/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/cosmos/cosmos-sdk/client/debug"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/client/keys"
"github.com/cosmos/cosmos-sdk/client/pruning"
"github.com/cosmos/cosmos-sdk/client/rpc"
"github.com/cosmos/cosmos-sdk/server"
serverconfig "github.com/cosmos/cosmos-sdk/server/config"
Expand Down Expand Up @@ -182,6 +183,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
NewTestnetCmd(simapp.ModuleBasics, banktypes.GenesisBalancesIterator{}),
debug.Cmd(),
config.Cmd(),
pruning.PruningCmd(newApp),
)

server.AddCommands(rootCmd, simapp.DefaultNodeHome, newApp, appExport, addModuleInitFlags)
Expand Down
33 changes: 22 additions & 11 deletions store/rootmulti/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func (rs *Store) StoreKeysByName() map[string]types.StoreKey {

// LoadLatestVersionAndUpgrade implements CommitMultiStore
func (rs *Store) LoadLatestVersionAndUpgrade(upgrades *types.StoreUpgrades) error {
ver := getLatestVersion(rs.db)
ver := GetLatestVersion(rs.db)
return rs.loadVersion(ver, upgrades)
}

Expand All @@ -173,7 +173,7 @@ func (rs *Store) LoadVersionAndUpgrade(ver int64, upgrades *types.StoreUpgrades)

// LoadLatestVersion implements CommitMultiStore.
func (rs *Store) LoadLatestVersion() error {
ver := getLatestVersion(rs.db)
ver := GetLatestVersion(rs.db)
return rs.loadVersion(ver, nil)
}

Expand Down Expand Up @@ -391,7 +391,7 @@ func (rs *Store) ListeningEnabled(key types.StoreKey) bool {
func (rs *Store) LastCommitID() types.CommitID {
if rs.lastCommitInfo == nil {
return types.CommitID{
Version: getLatestVersion(rs.db),
Version: GetLatestVersion(rs.db),
}
}

Expand Down Expand Up @@ -539,17 +539,28 @@ func (rs *Store) handlePruning(version int64) error {
}
rs.logger.Info("prune start", "height", version)
defer rs.logger.Info("prune end", "height", version)
return rs.pruneStores()
return rs.PruneStores(true, nil)
}

func (rs *Store) pruneStores() error {
pruningHeights, err := rs.pruningManager.GetFlushAndResetPruningHeights()
if err != nil {
return err
// PruneStores prunes the specific heights of the multi store.
// If clearPruningManager is true, the pruning manager will return the pruning heights,
// and they are appended to the pruningHeights to be pruned.
func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (err error) {
if clearPruningManager {
heights, err := rs.pruningManager.GetFlushAndResetPruningHeights()
if err != nil {
return err
}

if len(heights) == 0 {
rs.logger.Debug("no heights to be pruned from pruning manager")
}

pruningHeights = append(pruningHeights, heights...)
}

if len(pruningHeights) == 0 {
rs.logger.Debug("pruning skipped; no heights to prune")
rs.logger.Debug("no heights need to be pruned")
return nil
}

Expand Down Expand Up @@ -689,7 +700,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
if height == 0 {
return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0")
}
if height > uint64(getLatestVersion(rs.db)) {
if height > uint64(GetLatestVersion(rs.db)) {
return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height)
}

Expand Down Expand Up @@ -981,7 +992,7 @@ type storeParams struct {
initialVersion uint64
}

func getLatestVersion(db dbm.DB) int64 {
func GetLatestVersion(db dbm.DB) int64 {
bz, err := db.Get([]byte(latestVersionKey))
if err != nil {
panic(err)
Expand Down

0 comments on commit d874ace

Please sign in to comment.