diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 04cb64c35a..a1d476bb3c 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,15 +18,12 @@ package main import ( "bytes" - "encoding/json" "errors" - "os" "path/filepath" "time" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" @@ -171,56 +168,53 @@ It's also usable without snapshot enabled. func pruneBlock(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) - //defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, false) - // Make sure we have a valid genesis JSON - genesisPath := ctx.GlobalString(utils.GenesisFlag.Name) - if len(genesisPath) == 0 { - utils.Fatalf("Must supply path to genesis JSON file") - } - file, err := os.Open(genesisPath) - if err != nil { - utils.Fatalf("Failed to read genesis file: %v", err) - } - defer file.Close() + defer stack.Close() + + chaindb := utils.MakeChainDatabaseForBlockPrune(ctx, stack, false) + chaindb.Close() - genesis := new(core.Genesis) - if err := json.NewDecoder(file).Decode(genesis); err != nil { - utils.Fatalf("invalid genesis file: %v", err) - } - if err != nil { - utils.Fatalf("Failed to decode genesis: %v", err) - } freezer := config.Eth.DatabaseFreezer for _, name := range []string{"chaindata"} { - root := stack.ResolvePath(name) // /Users/user/storage/Private_BSC_Storage/build/bin/node/geth/chaindata + root := stack.ResolvePath(name) switch { case freezer == "": freezer = filepath.Join(root, "ancient") case !filepath.IsAbs(freezer): freezer = stack.ResolvePath(freezer) } - pruner, err := pruner.NewBlockPruner(chaindb, stack, stack.ResolvePath(""), freezer, genesis) + pruner, err := pruner.NewBlockPruner(chaindb, stack, stack.ResolvePath(""), freezer) if err != nil { utils.Fatalf("Failed to create block pruner", err) } backfreezer := filepath.Join(root, "ancient_back_up") - if err := pruner.BlockPruneBackUp(name, config.Eth.DatabaseCache, utils.MakeDatabaseHandles(), backfreezer, "", false); err != nil { + + if err := pruner.BlockPruneBackUp(name, config.Eth.DatabaseCache, utils.MakeDatabaseHandles(), backfreezer, freezer, "", false); err != nil { log.Error("Failed to back up block", "err", err) return err } } + log.Info("geth block offline pruning backup successfully") - oldAncientPath := ctx.GlobalString(utils.AncientFlag.Name) - newAncientPath := ctx.GlobalString(utils.AncientBackUpFlag.Name) - if err := pruner.BlockPrune(oldAncientPath, newAncientPath); err != nil { - utils.Fatalf("Failed to prune block", err) - return err + + if path := getAncientPath(ctx); path != "" { + oldAncientPath := path + "/ancient" + newAncientPath := path + "/ancient_back" + if err := pruner.BlockPrune(oldAncientPath, newAncientPath); err != nil { + utils.Fatalf("Failed to prune block", err) + return err + } + } else { + utils.Fatalf("Prune failed, did not specify the AncientPath %v") } + return nil } +func getAncientPath(ctx *cli.Context) string { + return ctx.GlobalString(utils.AncientFlag.Name) +} + func pruneState(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) defer stack.Close() diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b74e7e3a5f..596d1da675 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1912,6 +1912,28 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. return chainDb } +// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. +func MakeChainDatabaseForBlockPrune(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database { + var ( + cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 + handles = MakeDatabaseHandles() + + err error + chainDb ethdb.Database + ) + if ctx.GlobalString(SyncModeFlag.Name) == "light" { + name := "lightchaindata" + chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly) + } else { + name := "chaindata" + chainDb, err = stack.OpenDatabaseWithFreezerForPruneBlock(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) + } + if err != nil { + Fatalf("Could not open database: %v", err) + } + return chainDb +} + func MakeGenesis(ctx *cli.Context) *core.Genesis { var genesis *core.Genesis switch { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 2e9d95b2e2..7be10be1cb 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -294,6 +294,20 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu return nil // Can't find the data anywhere. } +func ReadOffSetOfAncientFreezer(db ethdb.KeyValueStore) uint64 { + offset, _ := db.Get(offSetOfAncientFreezer) + if offset == nil { + return 0 + } + return binary.BigEndian.Uint64(offset) +} + +func WriteOffSetOfAncientFreezer(db ethdb.KeyValueStore, offset uint64) { + if err := db.Put(offSetOfAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { + log.Crit("Failed to store offSetOfAncientFreezer", "err", err) + } +} + // HasHeader verifies the existence of a block header corresponding to the hash. func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Ancient(freezerHashTable, number); err == nil && common.BytesToHash(has) == hash { @@ -724,7 +738,6 @@ func WriteAncientBlock(db ethdb.AncientWriter, block *types.Block, receipts type return len(headerBlob) + len(bodyBlob) + len(receiptBlob) + len(tdBlob) + common.HashLength } - // DeleteBlock removes all block data associated with a hash. func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { DeleteReceipts(db, hash, number) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index a3cc66231f..da15cfc161 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -122,11 +122,6 @@ func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, return errNotSupported } -// AppendAncientNoBody returns an error as we don't have a backing chain freezer. -func (db *nofreezedb) AppendAncientNoBody(number uint64, hash, header, receipts, td []byte) error { - return errNotSupported -} - // TruncateAncients returns an error as we don't have a backing chain freezer. func (db *nofreezedb) TruncateAncients(items uint64) error { return errNotSupported @@ -236,6 +231,47 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st }, nil } +// NewDatabaseWithFreezer creates a high level database on top of a given key- +// value data store with a freezer moving immutable chain segments into cold +// storage. +//Without goroutine of freeze running +func NewDatabaseWithFreezerForPruneBlock(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { + // Create the idle freezer instance + frdb, err := newFreezer(freezer, namespace, readonly) + if err != nil { + return nil, err + } + + return &freezerdb{ + KeyValueStore: db, + AncientStore: frdb, + }, nil +} + +// NewDatabaseWithFreezer creates a high level database on top of a given key- +// value data store with a freezer moving immutable chain segments into cold +// storage. +func NewDatabaseWithFreezerBackup(offset uint64, db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { + // Create the idle freezer instance + frdb, err := newFreezer(freezer, namespace, readonly) + if err != nil { + return nil, err + } + + //Assign the new offset to the new backup freezer while creating freezer + frdb.offset = offset + + // Freezer is consistent with the key-value database, permit combining the two + if !frdb.readonly { + go frdb.freeze(db) + } + + return &freezerdb{ + KeyValueStore: db, + AncientStore: frdb, + }, nil +} + // NewMemoryDatabase creates an ephemeral in-memory key-value database without a // freezer moving immutable chain segments into cold storage. func NewMemoryDatabase() ethdb.Database { @@ -274,6 +310,49 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer return frdb, nil } +// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a +// freezer moving immutable chain segments into cold storage. +func NewLevelDBDatabaseWithFreezerForPruneBlock(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { + kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + frdb, err := NewDatabaseWithFreezerForPruneBlock(kvdb, freezer, namespace, readonly) + if err != nil { + kvdb.Close() + return nil, err + } + return frdb, nil +} + +// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a +// freezer moving immutable chain segments into cold storage. +func NewLevelDBDatabaseWithFreezerBackup(offset uint64, file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { + kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + db, err := NewDatabaseWithFreezerBackup(offset, kvdb, freezer, namespace, readonly) + if err != nil { + kvdb.Close() + return nil, err + } + + return db, nil +} + +func ReOpenDatabaseWithFreezerBackup(frdb ethdb.AncientStore, file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + + return &freezerdb{ + KeyValueStore: kvdb, + AncientStore: frdb, + }, nil +} + type counter uint64 func (c counter) String() string { diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 94b99a64eb..14f14f98ff 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -85,6 +85,8 @@ type freezer struct { quit chan struct{} closeOnce sync.Once + + offset uint64 } // newFreezer creates a chain freezer that moves ancient chain data into @@ -164,7 +166,7 @@ func (f *freezer) Close() error { // in the freezer. func (f *freezer) HasAncient(kind string, number uint64) (bool, error) { if table := f.tables[kind]; table != nil { - return table.has(number), nil + return table.has(number - f.offset), nil } return false, nil } @@ -172,7 +174,7 @@ func (f *freezer) HasAncient(kind string, number uint64) (bool, error) { // Ancient retrieves an ancient binary blob from the append-only immutable files. func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) { if table := f.tables[kind]; table != nil { - return table.Retrieve(number) + return table.Retrieve(number - f.offset) } return nil, errUnknownTable } @@ -201,7 +203,7 @@ func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td return errReadOnly } // Ensure the binary blobs we are appending is continuous with freezer. - if atomic.LoadUint64(&f.frozen) != number { + if atomic.LoadUint64(&f.frozen) != number-f.offset { return errOutOrderInsertion } // Rollback all inserted data if any insertion below failed to ensure @@ -313,6 +315,11 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { continue } number := ReadHeaderNumber(nfdb, hash) + + //minus the freezer offset + if number != nil { + *number = *number - f.offset + } threshold := atomic.LoadUint64(&f.threshold) switch { diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index b4fb99e451..4187966b88 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -69,6 +69,7 @@ var ( // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") + offSetOfAncientFreezer = []byte("OffSetOfAncientFreezer") // badBlockKey tracks the list of bad blocks seen by local badBlockKey = []byte("InvalidBlock") diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 437eec52a6..00664acc35 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -93,9 +92,7 @@ type BlockPruner struct { db ethdb.Database chaindbDir string ancientdbDir string - headHeader *types.Header n *node.Node - genesis *core.Genesis } // NewPruner creates the pruner instance. @@ -128,14 +125,13 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize, trie }, nil } -func NewBlockPruner(db ethdb.Database, n *node.Node, chaindbDir, ancientdbDir string, genesis *core.Genesis) (*BlockPruner, error) { +func NewBlockPruner(db ethdb.Database, n *node.Node, chaindbDir, ancientdbDir string) (*BlockPruner, error) { return &BlockPruner{ db: db, chaindbDir: chaindbDir, ancientdbDir: ancientdbDir, n: n, - genesis: genesis, }, nil } @@ -258,54 +254,67 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta } // Prune block body data -func (p *BlockPruner) BlockPruneBackUp(name string, cache, handles int, backFreezer, namespace string, readonly bool) error { +func (p *BlockPruner) BlockPruneBackUp(name string, cache, handles int, backFreezer, freezer, namespace string, readonly bool) error { //Back-up the necessary data within original ancient directory, create new freezer backup directory backFreezer - //db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, backFreezer, namespace, readonly) + return nil start := time.Now() - chainDbBack, err := p.n.OpenDatabaseWithFreezer(name, cache, handles, backFreezer, namespace, readonly) - if err != nil { - log.Error("Failed to open ancient database: %v", err) - return err - } - - //write back-up data to new chainDb - //write genesis block firstly - genesis := p.genesis - if _, _, err := core.SetupGenesisBlock(chainDbBack, genesis); err != nil { - log.Error("Failed to write genesis block: %v", err) - return err - } //write the latest 128 blocks data of the ancient db // If we can't access the freezer or it's empty, abort + oldOffSet := rawdb.ReadOffSetOfAncientFreezer(p.db) frozen, err := p.db.Ancients() + if err != nil || frozen == 0 { - return errors.New("Can't access the freezer or it's empty, abort") + return errors.New("can't access the freezer or it's empty, abort") } - start_index := frozen - 128 - if start_index < 0 { - start_index = 0 + startBlockNumber := frozen + oldOffSet - 128 + + newOffSet := oldOffSet + frozen - 128 + + //chainDb, err := p.n.OpenDatabaseWithFreezer(name, cache, handles, freezer, namespace, readonly) + chainDb, err := p.n.OpenDatabaseWithFreezerForPruneBlock(name, cache, handles, freezer, namespace, readonly) + if err != nil { + log.Error("Failed to open ancient database: %v", err) + return err } + //write the new offset into db for new freezer usage + rawdb.WriteOffSetOfAncientFreezer(chainDb, newOffSet) + //chainDb.Close() + + blockList := make([]*types.Block, 0, 128) + receiptsList := make([]types.Receipts, 0, 128) + externTdList := make([]*big.Int, 0, 128) + //All ancient data within the most recent 128 blocks write into new ancient_back directory - chainDb := p.db - for blockNumber := start_index; blockNumber < frozen; blockNumber++ { + for blockNumber := startBlockNumber; blockNumber < frozen+oldOffSet; blockNumber++ { blockHash := rawdb.ReadCanonicalHash(chainDb, blockNumber) block := rawdb.ReadBlock(chainDb, blockHash, blockNumber) + blockList = append(blockList, block) receipts := rawdb.ReadRawReceipts(chainDb, blockHash, blockNumber) + receiptsList = append(receiptsList, receipts) // Calculate the total difficulty of the block td := rawdb.ReadTd(chainDb, blockHash, blockNumber) if td == nil { return consensus.ErrUnknownAncestor } externTd := new(big.Int).Add(block.Difficulty(), td) - rawdb.WriteAncientBlock(chainDbBack, block, receipts, externTd) + externTdList = append(externTdList, externTd) } - //chainDb.TruncateAncients(start_index - 1) chainDb.Close() - chainDbBack.Close() - log.Info("Block pruning BackUp successful", common.PrettyDuration(time.Since(start))) + chainDbBack, err := p.n.OpenDatabaseWithFreezerBackup(newOffSet, name, cache, handles, backFreezer, namespace, readonly) + if err != nil { + log.Error("Failed to open ancient database: %v", err) + return err + } + //Write into ancient_backup + for id := 0; id < len(blockList); id++ { + rawdb.WriteAncientBlock(chainDbBack, blockList[id], receiptsList[id], externTdList[id]) + } + + chainDbBack.Close() + log.Info("Block pruning BackUp successfully", common.PrettyDuration(time.Since(start))) return nil } @@ -313,13 +322,12 @@ func BlockPrune(oldAncientPath, newAncientPath string) error { //Delete directly for the old ancientdb, e.g.: path ../chaindb/ancient if err := os.RemoveAll(oldAncientPath); err != nil { log.Error("Failed to remove old ancient directory %v", err) - return err } //Rename the new ancientdb path same to the old if err := os.Rename(newAncientPath, oldAncientPath); err != nil { - log.Error("Failed to rename new ancient directory %v", err) + log.Error("Failed to rename new ancient directory") return err } return nil diff --git a/ethdb/database.go b/ethdb/database.go index c399f45c20..40d0e01d57 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -94,7 +94,6 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error - } // Reader contains the methods required to read data from both key-value as well as diff --git a/node/node.go b/node/node.go index c122dad32c..aff6b88e92 100644 --- a/node/node.go +++ b/node/node.go @@ -634,6 +634,72 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, return db, err } +// OpenDatabaseWithFreezer opens an existing database with the given name (or +// creates one if no previous can be found) from within the node's data directory, +// also attaching a chain freezer to it that moves ancient chain data from the +// database to immutable append-only files. If the node is an ephemeral one, a +// memory database is returned. +func (n *Node) OpenDatabaseWithFreezerForPruneBlock(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { + n.lock.Lock() + defer n.lock.Unlock() + if n.state == closedState { + return nil, ErrNodeStopped + } + + var db ethdb.Database + var err error + if n.config.DataDir == "" { + db = rawdb.NewMemoryDatabase() + } else { + root := n.ResolvePath(name) + switch { + case freezer == "": + freezer = filepath.Join(root, "ancient") + case !filepath.IsAbs(freezer): + freezer = n.ResolvePath(freezer) + } + db, err = rawdb.NewLevelDBDatabaseWithFreezerForPruneBlock(root, cache, handles, freezer, namespace, readonly) + } + + if err == nil { + db = n.wrapDatabase(db) + } + return db, err +} + +// OpenDatabaseWithFreezer opens an existing database with the given name (or +// creates one if no previous can be found) from within the node's data directory, +// also attaching a chain freezer to it that moves ancient chain data from the +// database to immutable append-only files. If the node is an ephemeral one, a +// memory database is returned. +func (n *Node) OpenDatabaseWithFreezerBackup(offset uint64, name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { + n.lock.Lock() + defer n.lock.Unlock() + if n.state == closedState { + return nil, ErrNodeStopped + } + + var db ethdb.Database + var err error + if n.config.DataDir == "" { + db = rawdb.NewMemoryDatabase() + } else { + root := n.ResolvePath(name) + switch { + case freezer == "": + freezer = filepath.Join(root, "ancient") + case !filepath.IsAbs(freezer): + freezer = n.ResolvePath(freezer) + } + db, err = rawdb.NewLevelDBDatabaseWithFreezerBackup(offset, root, cache, handles, freezer, namespace, readonly) + } + + if err == nil { + db = n.wrapDatabase(db) + } + return db, err +} + func (n *Node) OpenDiffDatabase(name string, handles int, diff, namespace string, readonly bool) (*leveldb.Database, error) { n.lock.Lock() defer n.lock.Unlock()