update

cmd/geth/snapshot.go

@@ -18,15 +18,12 @@ package main
 
 import (
  "bytes"
- "encoding/json"
  "errors"
- "os"
  "path/filepath"
  "time"
 
  "github.com/ethereum/go-ethereum/cmd/utils"
  "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/core"
  "github.com/ethereum/go-ethereum/core/rawdb"
  "github.com/ethereum/go-ethereum/core/state"
  "github.com/ethereum/go-ethereum/core/state/pruner"
@@ -171,56 +168,53 @@ It's also usable without snapshot enabled.
 
 func pruneBlock(ctx *cli.Context) error {
  stack, config := makeConfigNode(ctx)
- //defer stack.Close()
- chaindb := utils.MakeChainDatabase(ctx, stack, false)
- // Make sure we have a valid genesis JSON
- genesisPath := ctx.GlobalString(utils.GenesisFlag.Name)
- if len(genesisPath) == 0 {
- utils.Fatalf("Must supply path to genesis JSON file")
- }
- file, err := os.Open(genesisPath)
- if err != nil {
- utils.Fatalf("Failed to read genesis file: %v", err)
- }
- defer file.Close()
+ defer stack.Close()
+
+ chaindb := utils.MakeChainDatabaseForBlockPrune(ctx, stack, false)
+ chaindb.Close()
 
- genesis := new(core.Genesis)
- if err := json.NewDecoder(file).Decode(genesis); err != nil {
- utils.Fatalf("invalid genesis file: %v", err)
- }
- if err != nil {
- utils.Fatalf("Failed to decode genesis: %v", err)
- }
  freezer := config.Eth.DatabaseFreezer
 
  for _, name := range []string{"chaindata"} {
- root := stack.ResolvePath(name) // /Users/user/storage/Private_BSC_Storage/build/bin/node/geth/chaindata
+ root := stack.ResolvePath(name)
  switch {
  case freezer == "":
  freezer = filepath.Join(root, "ancient")
  case !filepath.IsAbs(freezer):
  freezer = stack.ResolvePath(freezer)
  }
- pruner, err := pruner.NewBlockPruner(chaindb, stack, stack.ResolvePath(""), freezer, genesis)
+ pruner, err := pruner.NewBlockPruner(chaindb, stack, stack.ResolvePath(""), freezer)
  if err != nil {
  utils.Fatalf("Failed to create block pruner", err)
  }
  backfreezer := filepath.Join(root, "ancient_back_up")
- if err := pruner.BlockPruneBackUp(name, config.Eth.DatabaseCache, utils.MakeDatabaseHandles(), backfreezer, "", false); err != nil {
+
+ if err := pruner.BlockPruneBackUp(name, config.Eth.DatabaseCache, utils.MakeDatabaseHandles(), backfreezer, freezer, "", false); err != nil {
  log.Error("Failed to back up block", "err", err)
  return err
  }
  }
+
  log.Info("geth block offline pruning backup successfully")
- oldAncientPath := ctx.GlobalString(utils.AncientFlag.Name)
- newAncientPath := ctx.GlobalString(utils.AncientBackUpFlag.Name)
- if err := pruner.BlockPrune(oldAncientPath, newAncientPath); err != nil {
- utils.Fatalf("Failed to prune block", err)
- return err
+
+ if path := getAncientPath(ctx); path != "" {
+ oldAncientPath := path + "/ancient"
+ newAncientPath := path + "/ancient_back"
+ if err := pruner.BlockPrune(oldAncientPath, newAncientPath); err != nil {
+ utils.Fatalf("Failed to prune block", err)
+ return err
+ }
+ } else {
+ utils.Fatalf("Prune failed, did not specify the AncientPath %v")
  }
+
  return nil
 }
 
+func getAncientPath(ctx *cli.Context) string {
+ return ctx.GlobalString(utils.AncientFlag.Name)
+}
+
 func pruneState(ctx *cli.Context) error {
  stack, config := makeConfigNode(ctx)
  defer stack.Close()

cmd/utils/flags.go

@@ -1912,6 +1912,28 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
  return chainDb
 }
 
+// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
+func MakeChainDatabaseForBlockPrune(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database {
+ var (
+ cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
+ handles = MakeDatabaseHandles()
+
+ err error
+ chainDb ethdb.Database
+ )
+ if ctx.GlobalString(SyncModeFlag.Name) == "light" {
+ name := "lightchaindata"
+ chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly)
+ } else {
+ name := "chaindata"
+ chainDb, err = stack.OpenDatabaseWithFreezerForPruneBlock(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
+ }
+ if err != nil {
+ Fatalf("Could not open database: %v", err)
+ }
+ return chainDb
+}
+
 func MakeGenesis(ctx *cli.Context) *core.Genesis {
  var genesis *core.Genesis
  switch {

core/rawdb/accessors_chain.go

@@ -294,6 +294,20 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu
  return nil // Can't find the data anywhere.
 }
 
+func ReadOffSetOfAncientFreezer(db ethdb.KeyValueStore) uint64 {
+ offset, _ := db.Get(offSetOfAncientFreezer)
+ if offset == nil {
+ return 0
+ }
+ return binary.BigEndian.Uint64(offset)
+}
+
+func WriteOffSetOfAncientFreezer(db ethdb.KeyValueStore, offset uint64) {
+ if err := db.Put(offSetOfAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil {
+ log.Crit("Failed to store offSetOfAncientFreezer", "err", err)
+ }
+}
+
 // HasHeader verifies the existence of a block header corresponding to the hash.
 func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool {
  if has, err := db.Ancient(freezerHashTable, number); err == nil && common.BytesToHash(has) == hash {
@@ -724,7 +738,6 @@ func WriteAncientBlock(db ethdb.AncientWriter, block *types.Block, receipts type
  return len(headerBlob) + len(bodyBlob) + len(receiptBlob) + len(tdBlob) + common.HashLength
 }
 
-
 // DeleteBlock removes all block data associated with a hash.
 func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
  DeleteReceipts(db, hash, number)

core/rawdb/database.go

@@ -122,11 +122,6 @@ func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts,
  return errNotSupported
 }
 
-// AppendAncientNoBody returns an error as we don't have a backing chain freezer.
-func (db *nofreezedb) AppendAncientNoBody(number uint64, hash, header, receipts, td []byte) error {
- return errNotSupported
-}
-
 // TruncateAncients returns an error as we don't have a backing chain freezer.
 func (db *nofreezedb) TruncateAncients(items uint64) error {
  return errNotSupported
@@ -236,6 +231,47 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
  }, nil
 }
 
+// NewDatabaseWithFreezer creates a high level database on top of a given key-
+// value data store with a freezer moving immutable chain segments into cold
+// storage.
+//Without goroutine of freeze running
+func NewDatabaseWithFreezerForPruneBlock(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+ // Create the idle freezer instance
+ frdb, err := newFreezer(freezer, namespace, readonly)
+ if err != nil {
+ return nil, err
+ }
+
+ return &freezerdb{
+ KeyValueStore: db,
+ AncientStore: frdb,
+ }, nil
+}
+
+// NewDatabaseWithFreezer creates a high level database on top of a given key-
+// value data store with a freezer moving immutable chain segments into cold
+// storage.
+func NewDatabaseWithFreezerBackup(offset uint64, db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+ // Create the idle freezer instance
+ frdb, err := newFreezer(freezer, namespace, readonly)
+ if err != nil {
+ return nil, err
+ }
+
+ //Assign the new offset to the new backup freezer while creating freezer
+ frdb.offset = offset
+
+ // Freezer is consistent with the key-value database, permit combining the two
+ if !frdb.readonly {
+ go frdb.freeze(db)
+ }
+
+ return &freezerdb{
+ KeyValueStore: db,
+ AncientStore: frdb,
+ }, nil
+}
+
 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a
 // freezer moving immutable chain segments into cold storage.
 func NewMemoryDatabase() ethdb.Database {
@@ -274,6 +310,49 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
  return frdb, nil
 }
 
+// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
+// freezer moving immutable chain segments into cold storage.
+func NewLevelDBDatabaseWithFreezerForPruneBlock(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+ kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+ if err != nil {
+ return nil, err
+ }
+ frdb, err := NewDatabaseWithFreezerForPruneBlock(kvdb, freezer, namespace, readonly)
+ if err != nil {
+ kvdb.Close()
+ return nil, err
+ }
+ return frdb, nil
+}
+
+// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
+// freezer moving immutable chain segments into cold storage.
+func NewLevelDBDatabaseWithFreezerBackup(offset uint64, file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+ kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+ if err != nil {
+ return nil, err
+ }
+ db, err := NewDatabaseWithFreezerBackup(offset, kvdb, freezer, namespace, readonly)
+ if err != nil {
+ kvdb.Close()
+ return nil, err
+ }
+
+ return db, nil
+}
+
+func ReOpenDatabaseWithFreezerBackup(frdb ethdb.AncientStore, file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+ kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+ if err != nil {
+ return nil, err
+ }
+
+ return &freezerdb{
+ KeyValueStore: kvdb,
+ AncientStore: frdb,
+ }, nil
+}
+
 type counter uint64
 
 func (c counter) String() string {

core/rawdb/freezer.go

@@ -85,6 +85,8 @@ type freezer struct {
 
  quit chan struct{}
  closeOnce sync.Once
+
+ offset uint64
 }
 
 // newFreezer creates a chain freezer that moves ancient chain data into
@@ -164,15 +166,15 @@ func (f *freezer) Close() error {
 // in the freezer.
 func (f *freezer) HasAncient(kind string, number uint64) (bool, error) {
  if table := f.tables[kind]; table != nil {
- return table.has(number), nil
+ return table.has(number - f.offset), nil
  }
  return false, nil
 }
 
 // Ancient retrieves an ancient binary blob from the append-only immutable files.
 func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) {
  if table := f.tables[kind]; table != nil {
- return table.Retrieve(number)
+ return table.Retrieve(number - f.offset)
  }
  return nil, errUnknownTable
 }
@@ -201,7 +203,7 @@ func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td
  return errReadOnly
  }
  // Ensure the binary blobs we are appending is continuous with freezer.
- if atomic.LoadUint64(&f.frozen) != number {
+ if atomic.LoadUint64(&f.frozen) != number-f.offset {
  return errOutOrderInsertion
  }
  // Rollback all inserted data if any insertion below failed to ensure
@@ -313,6 +315,11 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) {
  continue
  }
  number := ReadHeaderNumber(nfdb, hash)
+
+ //minus the freezer offset
+ if number != nil {
+ *number = *number - f.offset
+ }
  threshold := atomic.LoadUint64(&f.threshold)
 
  switch {

core/rawdb/schema.go

@@ -69,6 +69,7 @@ var (
  // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync.
  fastTxLookupLimitKey = []byte("FastTransactionLookupLimit")
 
+ offSetOfAncientFreezer = []byte("OffSetOfAncientFreezer")
  // badBlockKey tracks the list of bad blocks seen by local
  badBlockKey = []byte("InvalidBlock")