[R4R]offline block prune (#543)

* offline block prune * update * update * update and add unit test * addressed comments from walt * Addressed comments from walt and Igor * ensure MPT and snapshot matched * add one more parameter to indicate blockprune * update the logic of creating freezerDb * update flag command description * expose the function for db inspect the offset/startBlockNumber * add flags to inspect prune info * rename flag of reserved-recent-blocks to block-amount-reserved * addressed comments from walt * handle the case of command interruption * refined goimports * addressed comments from walt * change the logic as restarting prune after interruption * addressed comments * reclaimed freezer logic * introduce flag to enable/disable check between MPT and snapshot * update the logic of frozen field in freezerDB * update the code in all places related to freezer change * addressed comments from dylan * update the logic for backup block difficulty * addressed comments from dylan
bnb-chain · Mar 1, 2022 · bce4e3c · bce4e3c
1 parent 27f0bfa
commit bce4e3c
Show file tree

Hide file tree

Showing 24 changed files with 877 additions and 64 deletions.
diff --git a/.gitignore b/.gitignore
@@ -48,3 +48,5 @@ profile.cov
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log
+cmd/geth/node/
+cmd/geth/__debug_bin
diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go
@@ -471,7 +471,7 @@ func importPreimages(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, false)
+ db := utils.MakeChainDatabase(ctx, stack, false, false)
  start := time.Now()
 
  if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
@@ -489,7 +489,7 @@ func exportPreimages(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  start := time.Now()
 
  if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
@@ -500,7 +500,7 @@ func exportPreimages(ctx *cli.Context) error {
 }
 
 func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  var header *types.Header
  if ctx.NArg() > 1 {
  return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())

diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go
@@ -72,6 +72,7 @@ Remove blockchain and state databases`,
  dbImportCmd,
  dbExportCmd,
  dbMetadataCmd,
+ ancientInspectCmd,
  },
  }
  dbInspectCmd = cli.Command{
@@ -251,6 +252,16 @@ WARNING: This is a low-level operation which may cause database corruption!`,
  },
  Description: "Shows metadata about the chain status.",
  }
+ ancientInspectCmd = cli.Command{
+ Action: utils.MigrateFlags(ancientInspect),
+ Name: "inspect-reserved-oldest-blocks",
+ Flags: []cli.Flag{
+ utils.DataDirFlag,
+ },
+ Usage: "Inspect the ancientStore information",
+ Description: `This commands will read current offset from kvdb, which is the current offset and starting BlockNumber
+of ancientStore, will also displays the reserved number of blocks in ancientStore `,
+ }
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -338,12 +349,21 @@ func inspect(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  defer db.Close()
 
  return rawdb.InspectDatabase(db, prefix, start)
 }
 
+func ancientInspect(ctx *cli.Context) error {
+ stack, _ := makeConfigNode(ctx)
+ defer stack.Close()
+
+ db := utils.MakeChainDatabase(ctx, stack, true, true)
+ defer db.Close()
+ return rawdb.AncientInspect(db)
+}
+
 func showLeveldbStats(db ethdb.Stater) {
  if stats, err := db.Stat("leveldb.stats"); err != nil {
  log.Warn("Failed to read database stats", "error", err)
@@ -361,7 +381,7 @@ func dbStats(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  defer db.Close()
 
  showLeveldbStats(db)
@@ -372,7 +392,7 @@ func dbCompact(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, false)
+ db := utils.MakeChainDatabase(ctx, stack, false, false)
  defer db.Close()
 
  log.Info("Stats before compaction")
@@ -396,7 +416,7 @@ func dbGet(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  defer db.Close()
 
  key, err := parseHexOrString(ctx.Args().Get(0))
@@ -422,7 +442,7 @@ func dbDelete(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, false)
+ db := utils.MakeChainDatabase(ctx, stack, false, false)
  defer db.Close()
 
  key, err := parseHexOrString(ctx.Args().Get(0))
@@ -449,7 +469,7 @@ func dbPut(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, false)
+ db := utils.MakeChainDatabase(ctx, stack, false, false)
  defer db.Close()
 
  var (
@@ -483,7 +503,7 @@ func dbDumpTrie(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
 
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  defer db.Close()
  var (
  root []byte
@@ -604,7 +624,7 @@ func importLDBdata(ctx *cli.Context) error {
  }
  close(stop)
  }()
- db := utils.MakeChainDatabase(ctx, stack, false)
+ db := utils.MakeChainDatabase(ctx, stack, false, false)
  return utils.ImportLDBData(db, fName, int64(start), stop)
 }
 
@@ -700,14 +720,14 @@ func exportChaindata(ctx *cli.Context) error {
  }
  close(stop)
  }()
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
 
 func showMetaData(ctx *cli.Context) error {
  stack, _ := makeConfigNode(ctx)
  defer stack.Close()
- db := utils.MakeChainDatabase(ctx, stack, true)
+ db := utils.MakeChainDatabase(ctx, stack, true, false)
  ancients, err := db.Ancients()
  if err != nil {
  fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)

diff --git a/cmd/geth/main.go b/cmd/geth/main.go
@@ -166,6 +166,8 @@ var (
  utils.GpoIgnoreGasPriceFlag,
  utils.MinerNotifyFullFlag,
  configFileFlag,
+ utils.BlockAmountReserved,
+ utils.CheckSnapshotWithMPT,
  }
 
  rpcFlags = []cli.Flag{