From c9ef24f6ab17d1c0de3d818dad581b599bb1239e Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 10 Feb 2023 14:25:22 +0800 Subject: [PATCH 01/53] core/state: typo Signed-off-by: Delweng --- core/state/pruner/pruner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 4e3daac669..946f0c52c8 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -66,9 +66,9 @@ var ( // Pruner is an offline tool to prune the stale state with the // help of the snapshot. The workflow of pruner is very simple: // -// - iterate the snapshot, reconstruct the relevant state -// - iterate the database, delete all other state entries which -// don't belong to the target state and the genesis state +// - iterate the snapshot, reconstruct the relevant state +// - iterate the database, delete all other state entries which +// don't belong to the target state and the genesis state // // It can take several hours(around 2 hours for mainnet) to finish // the whole pruning work. It's recommended to run this offline tool @@ -489,7 +489,7 @@ const warningLog = ` WARNING! -The clean trie cache is not found. Please delete it by yourself after the +The clean trie cache is not found. Please delete it by yourself after the pruning. Remember don't start the Geth without deleting the clean trie cache otherwise the entire database may be damaged! From a4b592c24e6a506d1f2448a36f5597f62b72c261 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 17 Feb 2023 17:42:43 +0800 Subject: [PATCH 02/53] core/rawdb: backport from https://github.com/bnb-chain/bsc/pull/543 Signed-off-by: Delweng --- core/rawdb/accessors_chain_test.go | 6 +- core/rawdb/chain_iterator.go | 5 +- core/rawdb/database.go | 100 +++++++++++++++++++++++++++-- core/rawdb/freezer.go | 39 +++++++---- core/rawdb/schema.go | 6 ++ core/rawdb/table.go | 10 +++ 6 files changed, 143 insertions(+), 23 deletions(-) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 5d0b257aff..72bbaecd53 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -441,7 +441,7 @@ func TestAncientStorage(t *testing.T) { } defer os.RemoveAll(frdir) - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } @@ -582,7 +582,7 @@ func BenchmarkWriteAncientBlocks(b *testing.B) { b.Fatalf("failed to create temp freezer dir: %v", err) } defer os.RemoveAll(frdir) - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { b.Fatalf("failed to create database with ancient backend") } @@ -892,7 +892,7 @@ func TestHeadersRLPStorage(t *testing.T) { } defer os.Remove(frdir) - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 0d5a5ee6a7..6c79a57aee 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -34,7 +34,7 @@ import ( // injects into the database the block hash->number mappings. func InitDatabaseFromFreezer(db ethdb.Database) { // If we can't access the freezer or it's empty, abort - frozen, err := db.Ancients() + frozen, err := db.ItemAmountInAncient() if err != nil || frozen == 0 { return } @@ -43,8 +43,9 @@ func InitDatabaseFromFreezer(db ethdb.Database) { start = time.Now() logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log hash common.Hash + offset = db.AncientOffSet() ) - for i := uint64(0); i < frozen; { + for i := uint64(0) + offset; i < frozen+offset; { // We read 100K hashes at a time, for a total of 3.2M count := uint64(100_000) if i+count > frozen { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5d645b61db..ce4f1dce3d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -20,6 +20,7 @@ import ( "bytes" "errors" "fmt" + "math/big" "os" "sync/atomic" "time" @@ -124,6 +125,15 @@ func (db *nofreezedb) TruncateTail(items uint64) error { return errNotSupported } +// Ancients returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) ItemAmountInAncient() (uint64, error) { + return 0, errNotSupported +} + +func (db *nofreezedb) AncientOffSet() uint64 { + return 0 +} + // Sync returns an error as we don't have a backing chain freezer. func (db *nofreezedb) Sync() error { return errNotSupported @@ -157,15 +167,61 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } +func ReadOffSetOfCurrentAncientFreezer(db ethdb.KeyValueReader) uint64 { + offset, _ := db.Get(offSetOfCurrentAncientFreezer) + if offset == nil { + return 0 + } + return new(big.Int).SetBytes(offset).Uint64() +} + +func ReadOffSetOfLastAncientFreezer(db ethdb.KeyValueReader) uint64 { + offset, _ := db.Get(offSetOfLastAncientFreezer) + if offset == nil { + return 0 + } + return new(big.Int).SetBytes(offset).Uint64() +} + +func WriteOffSetOfCurrentAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { + if err := db.Put(offSetOfCurrentAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { + log.Crit("Failed to store offSetOfAncientFreezer", "err", err) + } +} +func WriteOffSetOfLastAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { + if err := db.Put(offSetOfLastAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { + log.Crit("Failed to store offSetOfAncientFreezer", "err", err) + } +} + +// NewFreezerDb only create a freezer without statedb. +func NewFreezerDb(db ethdb.KeyValueStore, frz, namespace string, readonly bool, newOffSet uint64) (*freezer, error) { + // Create the idle freezer instance, this operation should be atomic to avoid mismatch between offset and acientDB. + frdb, err := newFreezer(frz, namespace, readonly, newOffSet, freezerTableSize, FreezerNoSnappy) + if err != nil { + return nil, err + } + return frdb, nil +} + // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold // storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly, disableFreeze, isLastOffset bool) (ethdb.Database, error) { + var offset uint64 + // The offset of ancientDB should be handled differently in different scenarios. + if isLastOffset { + offset = ReadOffSetOfLastAncientFreezer(db) + } else { + offset = ReadOffSetOfCurrentAncientFreezer(db) + } + // Create the idle freezer instance - frdb, err := newFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy) + frdb, err := newFreezer(freezer, namespace, readonly, offset, freezerTableSize, FreezerNoSnappy) if err != nil { return nil, err } + // Since the freezer can be stored separately from the user's key-value database, // there's a fairly high probability that the user requests invalid combinations // of the freezer and database. Ensure that we don't shoot ourselves in the foot @@ -188,7 +244,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the genesis hash is empty, we have a new key-value store, so nothing to // validate in this method. If, however, the genesis hash is not nil, compare // it to the freezer content. - if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { + if kvgenesis, _ := db.Get(headerHashKey(0)); offset == 0 && len(kvgenesis) > 0 { if frozen, _ := frdb.Ancients(); frozen > 0 { // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both @@ -231,7 +287,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st } } // Freezer is consistent with the key-value database, permit combining the two - if !frdb.readonly { + if !disableFreeze && !frdb.readonly { frdb.wg.Add(1) go func() { frdb.freeze(db) @@ -269,12 +325,12 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a // freezer moving immutable chain segments into cold storage. -func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly, disableFreeze, isLastOffset bool) (ethdb.Database, error) { kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } - frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly) + frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly, disableFreeze, isLastOffset) if err != nil { kvdb.Close() return nil, err @@ -312,6 +368,36 @@ func (s *stat) Count() string { return s.count.String() } +func AncientInspect(db ethdb.Database) error { + offset := counter(ReadOffSetOfCurrentAncientFreezer(db)) + // Get number of ancient rows inside the freezer. + ancients := counter(0) + if count, err := db.ItemAmountInAncient(); err != nil { + log.Error("failed to get the items amount in ancientDB", "err", err) + return err + } else { + ancients = counter(count) + } + var endNumber counter + if offset+ancients <= 0 { + endNumber = 0 + } else { + endNumber = offset + ancients - 1 + } + stats := [][]string{ + {"Offset/StartBlockNumber", "Offset/StartBlockNumber of ancientDB", offset.String()}, + {"Amount of remained items in AncientStore", "Remaining items of ancientDB", ancients.String()}, + {"The last BlockNumber within ancientDB", "The last BlockNumber", endNumber.String()}, + } + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Database", "Category", "Items"}) + table.SetFooter([]string{"", "AncientStore information", ""}) + table.AppendBulk(stats) + table.Render() + + return nil +} + // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { @@ -444,7 +530,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } // Get number of ancient rows inside the freezer ancients := counter(0) - if count, err := db.Ancients(); err == nil { + if count, err := db.ItemAmountInAncient(); err == nil { ancients = counter(count) } // Display the database statistic. diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index b0cfeb679f..a92ae5a248 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -70,10 +70,10 @@ const ( // freezer is a memory mapped append-only database to store immutable chain data // into flat files: // -// - The append only nature ensures that disk writes are minimized. -// - The memory mapping ensures we can max out system memory for caching without -// reserving it for go-ethereum. This would also reduce the memory requirements -// of Geth, and thus also GC overhead. +// - The append only nature ensures that disk writes are minimized. +// - The memory mapping ensures we can max out system memory for caching without +// reserving it for go-ethereum. This would also reduce the memory requirements +// of Geth, and thus also GC overhead. type freezer struct { // WARNING: The `frozen` field is accessed atomically. On 32 bit platforms, only // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, @@ -96,6 +96,8 @@ type freezer struct { quit chan struct{} wg sync.WaitGroup closeOnce sync.Once + + offset uint64 // Starting BlockNumber in current freezer } // newFreezer creates a chain freezer that moves ancient chain data into @@ -103,7 +105,7 @@ type freezer struct { // // The 'tables' argument defines the data tables. If the value of a map // entry is true, snappy compression is disabled for the table. -func newFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*freezer, error) { +func newFreezer(datadir string, namespace string, readonly bool, offset uint64, maxTableSize uint32, tables map[string]bool) (*freezer, error) { // Create the initial freezer object var ( readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) @@ -131,6 +133,7 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui instanceLock: lock, trigger: make(chan chan struct{}), quit: make(chan struct{}), + offset: offset, } // Create the tables. @@ -175,6 +178,10 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui return nil, err } + // Some blocks in ancientDB may have already been frozen and been pruned, so adding the offset to + // reprensent the absolute number of blocks already frozen. + freezer.frozen += offset + // Create the write batch. freezer.writeBatch = newFreezerBatch(freezer) @@ -211,7 +218,7 @@ func (f *freezer) Close() error { // in the freezer. func (f *freezer) HasAncient(kind string, number uint64) (bool, error) { if table := f.tables[kind]; table != nil { - return table.has(number), nil + return table.has(number - f.offset), nil } return false, nil } @@ -219,16 +226,16 @@ func (f *freezer) HasAncient(kind string, number uint64) (bool, error) { // Ancient retrieves an ancient binary blob from the append-only immutable files. func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) { if table := f.tables[kind]; table != nil { - return table.Retrieve(number) + return table.Retrieve(number - f.offset) } return nil, errUnknownTable } // AncientRange retrieves multiple items in sequence, starting from the index 'start'. // It will return -// - at most 'max' items, -// - at least 1 item (even if exceeding the maxByteSize), but will otherwise -// return as many items as fit into maxByteSize. +// - at most 'max' items, +// - at least 1 item (even if exceeding the maxByteSize), but will otherwise +// return as many items as fit into maxByteSize. func (f *freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { if table := f.tables[kind]; table != nil { return table.RetrieveItems(start, count, maxBytes) @@ -241,6 +248,16 @@ func (f *freezer) Ancients() (uint64, error) { return atomic.LoadUint64(&f.frozen), nil } +// ItemAmountInAncient returns the actual length of current ancientDB. +func (f *freezer) ItemAmountInAncient() (uint64, error) { + return atomic.LoadUint64(&f.frozen) - atomic.LoadUint64(&f.offset), nil +} + +// AncientOffSet returns the offset of current ancientDB. +func (f *freezer) AncientOffSet() uint64 { + return atomic.LoadUint64(&f.offset) +} + // Tail returns the number of first stored item in the freezer. func (f *freezer) Tail() (uint64, error) { return atomic.LoadUint64(&f.tail), nil @@ -333,7 +350,7 @@ func (f *freezer) TruncateTail(tail uint64) error { return nil } for _, table := range f.tables { - if err := table.truncateTail(tail); err != nil { + if err := table.truncateTail(tail - f.offset); err != nil { return err } } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 1533211fc7..cc9ef0e534 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -72,6 +72,12 @@ var ( // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") + // offSet of new updated ancientDB. + offSetOfCurrentAncientFreezer = []byte("offSetOfCurrentAncientFreezer") + + // offSet of the ancientDB before updated version. + offSetOfLastAncientFreezer = []byte("offSetOfLastAncientFreezer") + // badBlockKey tracks the list of bad blocks seen by local badBlockKey = []byte("InvalidBlock") diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 5eadf5f7c1..c691cc0910 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -95,6 +95,16 @@ func (t *table) ReadAncients(fn func(reader ethdb.AncientReader) error) (err err return t.db.ReadAncients(fn) } +// ItemAmountInAncient returns the actual length of current ancientDB. +func (t *table) ItemAmountInAncient() (uint64, error) { + return t.db.ItemAmountInAncient() +} + +// AncientOffSet returns the offset of current ancientDB. +func (t *table) AncientOffSet() uint64 { + return t.db.AncientOffSet() +} + // TruncateHead is a noop passthrough that just forwards the request to the underlying // database. func (t *table) TruncateHead(items uint64) error { From 8c35ecea093e71214a929ec975bd7c3ee57fa2eb Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 17 Feb 2023 18:14:21 +0800 Subject: [PATCH 03/53] eth,ethdb,node,core/state: backport from https://github.com/bnb-chain/bsc/pull/543 Signed-off-by: Delweng --- core/state/pruner/pruner.go | 213 +++++++++++++++++++++++++++++++++++ eth/downloader/downloader.go | 2 +- ethdb/database.go | 7 ++ node/node.go | 4 +- 4 files changed, 223 insertions(+), 3 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 946f0c52c8..0489c57c91 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -28,14 +28,17 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/prometheus/tsdb/fileutil" ) const ( @@ -335,6 +338,216 @@ func (p *Pruner) Prune(root common.Hash) error { return prune(p.snaptree, root, p.db, p.stateBloom, filterName, middleRoots, start) } +type BlockPruner struct { + db ethdb.Database + oldAncientPath string + newAncientPath string + node *node.Node + BlockAmountReserved uint64 +} + +func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientPath string, BlockAmountReserved uint64) *BlockPruner { + return &BlockPruner{ + db: db, + oldAncientPath: oldAncientPath, + newAncientPath: newAncientPath, + node: n, + BlockAmountReserved: BlockAmountReserved, + } +} + +func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace string, readonly, interrupt bool) error { + // Open old db wrapper. + chainDb, err := p.node.OpenDatabaseWithFreezer(name, cache, handles, p.oldAncientPath, namespace, readonly, true, interrupt) + if err != nil { + log.Error("Failed to open ancient database", "err=", err) + return err + } + defer chainDb.Close() + log.Info("chainDB opened successfully") + + // Get the number of items in old ancient db. + itemsOfAncient, err := chainDb.ItemAmountInAncient() + log.Info("the number of items in ancientDB is ", "itemsOfAncient", itemsOfAncient) + + // If we can't access the freezer or it's empty, abort. + if err != nil || itemsOfAncient == 0 { + log.Error("can't access the freezer or it's empty, abort") + return errors.New("can't access the freezer or it's empty, abort") + } + + // If the items in freezer is less than the block amount that we want to reserve, it is not enough, should stop. + if itemsOfAncient < p.BlockAmountReserved { + log.Error("the number of old blocks is not enough to reserve,", "ancient items", itemsOfAncient, "the amount specified", p.BlockAmountReserved) + return errors.New("the number of old blocks is not enough to reserve") + } + + var oldOffSet uint64 + if interrupt { + // The interrupt scecario within this function is specific for old and new ancientDB exsisted concurrently, + // should use last version of offset for oldAncientDB, because current offset is + // actually of the new ancientDB_Backup, but what we want is the offset of ancientDB being backup. + oldOffSet = rawdb.ReadOffSetOfLastAncientFreezer(chainDb) + } else { + // Using current version of ancientDB for oldOffSet because the db for backup is current version. + oldOffSet = rawdb.ReadOffSetOfCurrentAncientFreezer(chainDb) + } + log.Info("the oldOffSet is ", "oldOffSet", oldOffSet) + + // Get the start BlockNumber for pruning. + startBlockNumber := oldOffSet + itemsOfAncient - p.BlockAmountReserved + log.Info("new offset/new startBlockNumber is ", "new offset", startBlockNumber) + + // Create new ancientdb backup and record the new and last version of offset in kvDB as well. + // For every round, newoffset actually equals to the startBlockNumber in ancient backup db. + frdbBack, err := rawdb.NewFreezerDb(chainDb, p.newAncientPath, namespace, readonly, startBlockNumber) + if err != nil { + log.Error("Failed to create ancient freezer backup", "err=", err) + return err + } + defer frdbBack.Close() + + offsetBatch := chainDb.NewBatch() + rawdb.WriteOffSetOfCurrentAncientFreezer(offsetBatch, startBlockNumber) + rawdb.WriteOffSetOfLastAncientFreezer(offsetBatch, oldOffSet) + if err := offsetBatch.Write(); err != nil { + log.Crit("Failed to write offset into disk", "err", err) + } + + // It's guaranteed that the old/new offsets are updated as well as the new ancientDB are created if this flock exist. + lock, _, err := fileutil.Flock(filepath.Join(p.newAncientPath, "PRUNEFLOCKBACK")) + if err != nil { + log.Error("file lock error", "err", err) + return err + } + + log.Info("prune info", "old offset", oldOffSet, "number of items in ancientDB", itemsOfAncient, "amount to reserve", p.BlockAmountReserved) + log.Info("new offset/new startBlockNumber recorded successfully ", "new offset", startBlockNumber) + + start := time.Now() + // All ancient data after and including startBlockNumber should write into new ancientDB ancient_back. + for blockNumber := startBlockNumber; blockNumber < itemsOfAncient+oldOffSet; blockNumber++ { + blockHash := rawdb.ReadCanonicalHash(chainDb, blockNumber) + block := rawdb.ReadBlock(chainDb, blockHash, blockNumber) + receipts := rawdb.ReadRawReceipts(chainDb, blockHash, blockNumber) + borReceipts := []*types.Receipt{rawdb.ReadBorReceipt(chainDb, blockHash, blockNumber)} + + // Calculate the total difficulty of the block + td := rawdb.ReadTd(chainDb, blockHash, blockNumber) + if td == nil { + return consensus.ErrUnknownAncestor + } + // Write into new ancient_back db. + if _, err := rawdb.WriteAncientBlocks(frdbBack, []*types.Block{block}, []types.Receipts{receipts}, []types.Receipts{borReceipts}, td); err != nil { + log.Error("failed to write new ancient", "error", err) + return err + } + // Print the log every 5s for better trace. + if common.PrettyDuration(time.Since(start)) > common.PrettyDuration(5*time.Second) { + log.Info("block backup process running successfully", "current blockNumber for backup", blockNumber) + start = time.Now() + } + } + lock.Release() + log.Info("block back up done", "current start blockNumber in ancientDB", startBlockNumber) + return nil +} + +// Backup the ancient data for the old ancient db, i.e. the most recent 128 blocks in ancient db. +func (p *BlockPruner) BlockPruneBackUp(name string, cache, handles int, namespace string, readonly, interrupt bool) error { + start := time.Now() + + if err := p.backUpOldDb(name, cache, handles, namespace, readonly, interrupt); err != nil { + return err + } + + log.Info("Block pruning BackUp successfully", "time duration since start is", common.PrettyDuration(time.Since(start))) + return nil +} + +func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, namespace string, readonly bool) error { + log.Info("RecoverInterruption for block prune") + newExist, err := CheckFileExist(p.newAncientPath) + if err != nil { + log.Error("newAncientDb path error") + return err + } + + if newExist { + log.Info("New ancientDB_backup existed in interruption scenario") + flockOfAncientBack, err := CheckFileExist(filepath.Join(p.newAncientPath, "PRUNEFLOCKBACK")) + if err != nil { + log.Error("Failed to check flock of ancientDB_Back %v", err) + return err + } + + // Indicating both old and new ancientDB existed concurrently. + // Delete directly for the new ancientdb to prune from start, e.g.: path ../chaindb/ancient_backup + if err := os.RemoveAll(p.newAncientPath); err != nil { + log.Error("Failed to remove old ancient directory %v", err) + return err + } + if flockOfAncientBack { + // Indicating the oldOffset/newOffset have already been updated. + if err := p.BlockPruneBackUp(name, cache, handles, namespace, readonly, true); err != nil { + log.Error("Failed to prune") + return err + } + } else { + // Indicating the flock did not exist and the new offset did not be updated, so just handle this case as usual. + if err := p.BlockPruneBackUp(name, cache, handles, namespace, readonly, false); err != nil { + log.Error("Failed to prune") + return err + } + } + + if err := p.AncientDbReplacer(); err != nil { + log.Error("Failed to replace ancientDB") + return err + } + } else { + log.Info("New ancientDB_backup did not exist in interruption scenario") + // Indicating new ancientDB even did not be created, just prune starting at backup from startBlockNumber as usual, + // in this case, the new offset have not been written into kvDB. + if err := p.BlockPruneBackUp(name, cache, handles, namespace, readonly, false); err != nil { + log.Error("Failed to prune") + return err + } + if err := p.AncientDbReplacer(); err != nil { + log.Error("Failed to replace ancientDB") + return err + } + } + + return nil +} + +func CheckFileExist(path string) (bool, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + // Indicating the file didn't exist. + return false, nil + } + return true, err + } + return true, nil +} + +func (p *BlockPruner) AncientDbReplacer() error { + // Delete directly for the old ancientdb, e.g.: path ../chaindb/ancient + if err := os.RemoveAll(p.oldAncientPath); err != nil { + log.Error("Failed to remove old ancient directory %v", err) + return err + } + + // Rename the new ancientdb path same to the old + if err := os.Rename(p.newAncientPath, p.oldAncientPath); err != nil { + log.Error("Failed to rename new ancient directory") + return err + } + return nil +} + // RecoverPruning will resume the pruning procedure during the system restart. // This function is used in this case: user tries to prune state data, but the // system was interrupted midway because of crash or manual-kill. In this case diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 135defc0b9..462cd8b818 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -560,7 +560,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd * } else { d.ancientLimit = 0 } - frozen, _ := d.stateDB.Ancients() // Ignore the error here since light client can also hit here. + frozen, _ := d.stateDB.ItemAmountInAncient() // Ignore the error here since light client can also hit here. // If a part of blockchain data has already been written into active store, // disable the ancient style insertion explicitly. diff --git a/ethdb/database.go b/ethdb/database.go index 88c8de6a16..017d4288aa 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -93,6 +93,12 @@ type AncientReader interface { // AncientSize returns the ancient size of the specified category. AncientSize(kind string) (uint64, error) + + // ItemAmountInAncient returns the actual length of current ancientDB. + ItemAmountInAncient() (uint64, error) + + // AncientOffSet returns the offset of current ancientDB. + AncientOffSet() uint64 } // AncientBatchReader is the interface for 'batched' or 'atomic' reading. @@ -164,6 +170,7 @@ type AncientStore interface { // Database contains all the methods required by the high level database to not // only access the key-value data store but also the chain freezer. +// //go:generate mockgen -destination=../eth/filters/IDatabase.go -package=filters . Database type Database interface { Reader diff --git a/node/node.go b/node/node.go index e12bcf6675..b0f8770dc3 100644 --- a/node/node.go +++ b/node/node.go @@ -719,7 +719,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly, disableFreeze, isLastOffset bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -738,7 +738,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, case !filepath.IsAbs(freezer): freezer = n.ResolvePath(freezer) } - db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly, disableFreeze, isLastOffset) } if err == nil { From 2b8ca7363a7a68ba33c3a77c8b204de85a3383e4 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 17 Feb 2023 18:23:42 +0800 Subject: [PATCH 04/53] eth,core: backport from https://github.com/bnb-chain/bsc/pull/543 Signed-off-by: Delweng --- core/blockchain.go | 10 +++++++--- core/blockchain_test.go | 20 ++++++++++---------- core/rawdb/freezer_test.go | 8 ++++---- core/tests/blockchain_repair_test.go | 8 ++++---- core/tests/blockchain_sethead_test.go | 2 +- core/tests/blockchain_snapshot_test.go | 4 ++-- eth/backend.go | 2 +- eth/downloader/downloader_test.go | 2 +- 8 files changed, 30 insertions(+), 26 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 74fd4bfeda..d83bb134db 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -293,9 +293,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par rawdb.InitDatabaseFromFreezer(bc.db) // If ancient database is not empty, reconstruct all missing // indices in the background. - frozen, _ := bc.db.Ancients() + frozen, _ := bc.db.ItemAmountInAncient() if frozen > 0 { - txIndexBlock = frozen + txIndexBlock, _ = bc.db.Ancients() } } if err := bc.loadLastState(); err != nil { @@ -332,7 +332,11 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } // Ensure that a previous crash in SetHead doesn't leave extra ancients - if frozen, err := bc.db.Ancients(); err == nil && frozen > 0 { + if frozen, err := bc.db.ItemAmountInAncient(); err == nil && frozen > 0 { + frozen, err = bc.db.Ancients() + if err != nil { + return nil, err + } var ( needRewind bool low uint64 diff --git a/core/blockchain_test.go b/core/blockchain_test.go index fa6b61225e..83e6760521 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -796,7 +796,7 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -892,7 +892,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(dir) - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1764,7 +1764,7 @@ func TestBlockchainRecovery(t *testing.T) { } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -1835,7 +1835,7 @@ func TestInsertReceiptChainRollback(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2102,7 +2102,7 @@ func testInsertKnownChainData(t *testing.T, typ string) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(dir) - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2266,7 +2266,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(dir) - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2576,7 +2576,7 @@ func TestTransactionIndices(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2604,7 +2604,7 @@ func TestTransactionIndices(t *testing.T) { // Init block chain with external ancients, check all needed indices has been indexed. limit := []uint64{0, 32, 64, 128} for _, l := range limit { - ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2624,7 +2624,7 @@ func TestTransactionIndices(t *testing.T) { } // Reconstruct a block chain which only reserves HEAD-64 tx indices - ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } @@ -2703,7 +2703,7 @@ func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { t.Fatalf("failed to create temp freezer dir: %v", err) } defer os.Remove(frdir) - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index b3fd3059e7..3379736047 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -116,7 +116,7 @@ func TestFreezerModifyRollback(t *testing.T) { // Reopen and check that the rolled-back data doesn't reappear. tables := map[string]bool{"test": true} - f2, err := newFreezer(dir, "", false, 2049, tables) + f2, err := newFreezer(dir, "", false, 0, 2049, tables) if err != nil { t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err) } @@ -263,7 +263,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { defer os.RemoveAll(dir) // Open non-readonly freezer and fill individual tables // with different amount of data. - f, err := newFreezer(dir, "", false, 2049, tables) + f, err := newFreezer(dir, "", false, 0, 2049, tables) if err != nil { t.Fatal("can't open freezer", err) } @@ -286,7 +286,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { // Re-openening as readonly should fail when validating // table lengths. - f, err = newFreezer(dir, "", true, 2049, tables) + f, err = newFreezer(dir, "", true, 0, 2049, tables) if err == nil { t.Fatal("readonly freezer should fail with differing table lengths") } @@ -301,7 +301,7 @@ func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, strin } // note: using low max table size here to ensure the tests actually // switch between multiple files. - f, err := newFreezer(dir, "", false, 2049, tables) + f, err := newFreezer(dir, "", false, 0, 2049, tables) if err != nil { t.Fatal("can't open freezer", err) } diff --git a/core/tests/blockchain_repair_test.go b/core/tests/blockchain_repair_test.go index 9b166b7165..70bc408749 100644 --- a/core/tests/blockchain_repair_test.go +++ b/core/tests/blockchain_repair_test.go @@ -1781,7 +1781,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -1884,7 +1884,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { db.Close() // Start a new blockchain back up and see where the repair leads us - db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } @@ -1949,7 +1949,7 @@ func TestIssue23496(t *testing.T) { os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -2017,7 +2017,7 @@ func TestIssue23496(t *testing.T) { db.Close() // Start a new blockchain back up and see where the repair leads us - db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } diff --git a/core/tests/blockchain_sethead_test.go b/core/tests/blockchain_sethead_test.go index 6160dfb48f..d34af9bd69 100644 --- a/core/tests/blockchain_sethead_test.go +++ b/core/tests/blockchain_sethead_test.go @@ -1963,7 +1963,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } diff --git a/core/tests/blockchain_snapshot_test.go b/core/tests/blockchain_snapshot_test.go index 26ef58f99f..dd6ede5d86 100644 --- a/core/tests/blockchain_snapshot_test.go +++ b/core/tests/blockchain_snapshot_test.go @@ -68,7 +68,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*core.BlockChain, []*type } os.RemoveAll(datadir) - db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false) + db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -273,7 +273,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { db.Close() // Start a new blockchain back up and see where the repair leads us - newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "", false) + newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "", false, false, false) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } diff --git a/eth/backend.go b/eth/backend.go index 869566a7ac..4b3bba8c77 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -138,7 +138,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { ethashConfig.NotifyFull = config.Miner.NotifyFull // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) + chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false, false, false) if err != nil { return nil, err } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a9242fba5b..b52d7d367f 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -64,7 +64,7 @@ func newTester() *downloadTester { panic(err) } - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false) if err != nil { panic(err) } From 4b96e5c67b0ea8dc982d81e5032314d07a3b04e5 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 17 Feb 2023 20:16:02 +0800 Subject: [PATCH 05/53] cmd: open db with freeze disabled Signed-off-by: Delweng --- cmd/geth/chaincmd.go | 6 +++--- cmd/geth/dbcmd.go | 28 ++++++++++++++-------------- cmd/geth/snapshot.go | 12 ++++++------ cmd/utils/flags.go | 8 ++++---- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6dc58f4af6..6df5c3302a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -369,7 +369,7 @@ func importPreimages(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, false) + db := utils.MakeChainDatabase(ctx, stack, false, false) start := time.Now() if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil { @@ -387,7 +387,7 @@ func exportPreimages(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) start := time.Now() if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil { @@ -398,7 +398,7 @@ func exportPreimages(ctx *cli.Context) error { } func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) var header *types.Header if ctx.NArg() > 1 { return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 9784a3efb9..30b9ecc78b 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -128,7 +128,7 @@ Remove blockchain and state databases`, utils.CacheFlag, utils.CacheDatabaseFlag, }, - Description: `This command performs a database compaction. + Description: `This command performs a database compaction. WARNING: This operation may take a very long time to finish, and may cause database corruption if it is aborted during execution'!`, } @@ -166,7 +166,7 @@ corruption if it is aborted during execution'!`, utils.MumbaiFlag, utils.BorMainnetFlag, }, - Description: `This command deletes the specified database key from the database. + Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } dbPutCmd = cli.Command{ @@ -185,7 +185,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.MumbaiFlag, utils.BorMainnetFlag, }, - Description: `This command sets a given database key to the given value. + Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } dbGetSlotsCmd = cli.Command{ @@ -373,7 +373,7 @@ func inspect(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() return rawdb.InspectDatabase(db, prefix, start) @@ -396,7 +396,7 @@ func dbStats(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() showLeveldbStats(db) @@ -407,7 +407,7 @@ func dbCompact(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, false) + db := utils.MakeChainDatabase(ctx, stack, false, false) defer db.Close() log.Info("Stats before compaction") @@ -431,7 +431,7 @@ func dbGet(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() key, err := parseHexOrString(ctx.Args().Get(0)) @@ -457,7 +457,7 @@ func dbDelete(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, false) + db := utils.MakeChainDatabase(ctx, stack, false, false) defer db.Close() key, err := parseHexOrString(ctx.Args().Get(0)) @@ -484,7 +484,7 @@ func dbPut(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, false) + db := utils.MakeChainDatabase(ctx, stack, false, false) defer db.Close() var ( @@ -518,7 +518,7 @@ func dbDumpTrie(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() var ( root []byte @@ -639,7 +639,7 @@ func importLDBdata(ctx *cli.Context) error { } close(stop) }() - db := utils.MakeChainDatabase(ctx, stack, false) + db := utils.MakeChainDatabase(ctx, stack, false, false) return utils.ImportLDBData(db, fName, int64(start), stop) } @@ -735,14 +735,14 @@ func exportChaindata(ctx *cli.Context) error { } close(stop) }() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop) } func showMetaData(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) ancients, err := db.Ancients() if err != nil { fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err) @@ -793,7 +793,7 @@ func freezerMigrate(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, false) + db := utils.MakeChainDatabase(ctx, stack, false, false) defer db.Close() // Check first block for legacy receipt format diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index d8bdd3769d..d612d2f172 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -154,7 +154,7 @@ geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced trie node or contract code is missing. This command can be used for state integrity verification. The default checking target is the HEAD state. It's basically identical -to traverse-state, but the check granularity is smaller. +to traverse-state, but the check granularity is smaller. It's also usable without snapshot enabled. `, @@ -181,7 +181,7 @@ It's also usable without snapshot enabled. }, Description: ` This command is semantically equivalent to 'geth dump', but uses the snapshots -as the backend data source, making this command a lot faster. +as the backend data source, making this command a lot faster. The argument is interpreted as block number or hash. If none is provided, the latest block is used. @@ -195,7 +195,7 @@ func pruneState(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, false) + chaindb := utils.MakeChainDatabase(ctx, stack, false, false) pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) if err != nil { log.Error("Failed to open snapshot tree", "err", err) @@ -224,7 +224,7 @@ func verifyState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") @@ -262,7 +262,7 @@ func traverseState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") @@ -351,7 +351,7 @@ func traverseRawState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1772913c0e..ffa136273e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1806,7 +1806,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } // Check if we have an already initialized chain and fall back to // that if so. Otherwise we need to generate a new genesis spec. - chaindb := MakeChainDatabase(ctx, stack, readonly) + chaindb := MakeChainDatabase(ctx, stack, readonly, false) if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { cfg.Genesis = nil // fallback to db content } @@ -1966,7 +1966,7 @@ func SplitTagsFlag(tagsFlag string) map[string]string { } // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. -func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database { +func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database { var ( cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 handles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) @@ -1979,7 +1979,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly) } else { name := "chaindata" - chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) + chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly, disableFreeze, false) } if err != nil { Fatalf("Could not open database: %v", err) @@ -2020,7 +2020,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai Fatalf("Valid genesis file is required as argument: {}", err) } - chainDb = MakeChainDatabase(ctx, stack, false) // TODO(rjl493456442) support read-only database + chainDb = MakeChainDatabase(ctx, stack, false, false) // TODO(rjl493456442) support read-only database config, _, err := core.SetupGenesisBlock(chainDb, genesis) if err != nil { Fatalf("%v", err) From 4e254078e5d2593cf824341b1ad69e2c247fd7d8 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 17 Feb 2023 20:16:17 +0800 Subject: [PATCH 06/53] cli: snapshot prune-block Signed-off-by: Delweng --- internal/cli/chain.go | 4 +- internal/cli/command.go | 5 + internal/cli/snapshot.go | 279 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 285 insertions(+), 3 deletions(-) diff --git a/internal/cli/chain.go b/internal/cli/chain.go index 9a7e9e8537..2ed2c6d45b 100644 --- a/internal/cli/chain.go +++ b/internal/cli/chain.go @@ -28,9 +28,9 @@ func (c *ChainCommand) Help() string { return `Usage: bor chain This command groups actions to interact with the chain. - + Set the new head of the chain: - + $ bor chain sethead ` } diff --git a/internal/cli/command.go b/internal/cli/command.go index 95f7776df6..ff620798ff 100644 --- a/internal/cli/command.go +++ b/internal/cli/command.go @@ -199,6 +199,11 @@ func Commands() map[string]MarkDownCommandFactory { Meta: meta, }, nil }, + "snapshot prune-block": func() (MarkDownCommand, error) { + return &PruneBlockCommand{ + Meta: meta, + }, nil + }, } } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 3c8e4ec97d..8e4bf871ef 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -3,14 +3,23 @@ package cli import ( + "errors" + "fmt" + "os" + "path/filepath" "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/cli/flagset" "github.com/ethereum/go-ethereum/internal/cli/server" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/trie" + "github.com/prometheus/tsdb/fileutil" "github.com/mitchellh/cli" ) @@ -161,7 +170,7 @@ func (c *PruneStateCommand) Run(args []string) int { return 1 } - chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false) + chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false, false, false) if err != nil { c.UI.Error(err.Error()) @@ -181,3 +190,271 @@ func (c *PruneStateCommand) Run(args []string) int { return 0 } + +type PruneBlockCommand struct { + *Meta + + datadirAncient string + cache int + blockAmountReserved uint64 + triesInMemory int + checkSnapshotWithMPT bool +} + +// MarkDown implements cli.MarkDown interface +func (c *PruneBlockCommand) MarkDown() string { + items := []string{ + "# Prune block", + "The ```bor snapshot prune-block``` command will prune ancient blockchain offline", + c.Flags().MarkDown(), + } + + return strings.Join(items, "\n\n") +} + +// Help implements the cli.Command interface +func (c *PruneBlockCommand) Help() string { + return `Usage: bor snapshot prune-block + + This command will prune blockchain databases at the given datadir location` + c.Flags().Help() +} + +// Synopsis implements the cli.Command interface +func (c *PruneBlockCommand) Synopsis() string { + return "Prune ancient data" +} + +// Flags: datadir, datadir.ancient, cache.trie.journal, bloomfilter.size +func (c *PruneBlockCommand) Flags() *flagset.Flagset { + flags := c.NewFlagSet("prune-block") + + flags.StringFlag(&flagset.StringFlag{ + Name: "datadir.ancient", + Value: &c.datadirAncient, + Usage: "Path of the ancient data directory to store information", + Default: "", + }) + + flags.IntFlag(&flagset.IntFlag{ + Name: "cache", + Usage: "Megabytes of memory allocated to internal caching", + Value: &c.cache, + Default: 1024, + Group: "Cache", + }) + flags.Uint64Flag(&flagset.Uint64Flag{ + Name: "block-amount-reserved", + Usage: "Sets the expected remained amount of blocks for offline block prune", + Value: &c.blockAmountReserved, + Default: 1024, + }) + + flags.IntFlag(&flagset.IntFlag{ + Name: "cache.triesinmemory", + Usage: "Number of block states (tries) to keep in memory (default = 128)", + Value: &c.triesInMemory, + Default: 128, + }) + + flags.BoolFlag(&flagset.BoolFlag{ + Name: "check-snapshot-with-mpt", + Value: &c.checkSnapshotWithMPT, + Usage: "Path of the trie journal directory to store information", + }) + + return flags +} + +// Run implements the cli.Command interface +func (c *PruneBlockCommand) Run(args []string) int { + flags := c.Flags() + + if err := flags.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + datadir := c.dataDir + if datadir == "" { + c.UI.Error("datadir is required") + return 1 + } + + // Create the node + node, err := node.New(&node.Config{ + DataDir: datadir, + }) + + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + defer node.Close() + + dbHandles, err := server.MakeDatabaseHandles() + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, c.cache, dbHandles, c.datadirAncient, "", false, true, false) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + defer chaindb.Close() + + err = c.accessDb(node, chaindb) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + err = c.pruneBlock(node, chaindb, dbHandles) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + return 0 +} + +func (c *PruneBlockCommand) accessDb(stack *node.Node, chaindb ethdb.Database) error { + if !c.checkSnapshotWithMPT { + return nil + } + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + return errors.New("failed to load head block") + } + headHeader := headBlock.Header() + //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) + if err != nil { + log.Error("snaptree error", "err", err) + return err // The relevant snapshot(s) might not exist + } + + // Use the HEAD-(n-1) as the target root. The reason for picking it is: + // - in most of the normal cases, the related state is available + // - the probability of this layer being reorg is very low + + // Retrieve all snapshot layers from the current HEAD. + // In theory there are n difflayers + 1 disk layer present, + // so n diff layers are expected to be returned. + layers := snaptree.Snapshots(headHeader.Root, c.triesInMemory, true) + if len(layers) != c.triesInMemory { + // Reject if the accumulated diff layers are less than n. It + // means in most of normal cases, there is no associated state + // with bottom-most diff layer. + log.Error("snapshot layers != TriesInMemory", "err", err) + return fmt.Errorf("snapshot not old enough yet: need %d more blocks", c.triesInMemory-len(layers)) + } + // Use the bottom-most diff layer as the target + targetRoot := layers[len(layers)-1].Root() + + // Ensure the root is really present. The weak assumption + // is the presence of root can indicate the presence of the + // entire trie. + if blob := rawdb.ReadTrieNode(chaindb, targetRoot); len(blob) == 0 { + // The special case is for clique based networks(rinkeby, goerli + // and some other private networks), it's possible that two + // consecutive blocks will have same root. In this case snapshot + // difflayer won't be created. So HEAD-(n-1) may not paired with + // head-(n-1) layer. Instead the paired layer is higher than the + // bottom-most diff layer. Try to find the bottom-most snapshot + // layer with state available. + // + // Note HEAD is ignored. Usually there is the associated + // state available, but we don't want to use the topmost state + // as the pruning target. + var found bool + for i := len(layers) - 2; i >= 1; i-- { + if blob := rawdb.ReadTrieNode(chaindb, layers[i].Root()); len(blob) != 0 { + targetRoot = layers[i].Root() + found = true + log.Info("Selecting middle-layer as the pruning target", "root", targetRoot, "depth", i) + break + } + } + if !found { + if blob := rawdb.ReadTrieNode(chaindb, snaptree.DiskRoot()); len(blob) != 0 { + targetRoot = snaptree.DiskRoot() + found = true + log.Info("Selecting disk-layer as the pruning target", "root", targetRoot) + } + } + if !found { + if len(layers) > 0 { + log.Error("no snapshot paired state") + return errors.New("no snapshot paired state") + } + return fmt.Errorf("associated state[%x] is not present", targetRoot) + } + } else { + if len(layers) > 0 { + log.Info("Selecting bottom-most difflayer as the pruning target", "root", targetRoot, "height", headHeader.Number.Uint64()-uint64(len(layers)-1)) + } else { + log.Info("Selecting user-specified state as the pruning target", "root", targetRoot) + } + } + return nil +} + +func (c *PruneBlockCommand) pruneBlock(stack *node.Node, chaindb ethdb.Database, fdHandles int) error { + oldAncientPath := c.datadirAncient + if !filepath.IsAbs(oldAncientPath) { + oldAncientPath = stack.ResolvePath(oldAncientPath) + } + + path, _ := filepath.Split(oldAncientPath) + if path == "" { + return errors.New("prune failed, did not specify the AncientPath") + } + newAncientPath := filepath.Join(path, "ancient_back") + + blockpruner := pruner.NewBlockPruner(chaindb, stack, oldAncientPath, newAncientPath, c.blockAmountReserved) + + lock, exist, err := fileutil.Flock(filepath.Join(oldAncientPath, "PRUNEFLOCK")) + if err != nil { + log.Error("file lock error", "err", err) + return err + } + if exist { + defer lock.Release() + log.Info("file lock existed, waiting for prune recovery and continue", "err", err) + if err := blockpruner.RecoverInterruption("chaindata", c.cache, fdHandles, "", false); err != nil { + log.Error("Pruning failed", "err", err) + return err + } + log.Info("Block prune successfully") + return nil + } + + if _, err := os.Stat(newAncientPath); err == nil { + // No file lock found for old ancientDB but new ancientDB exsisted, indicating the geth was interrupted + // after old ancientDB removal, this happened after backup successfully, so just rename the new ancientDB + if err := blockpruner.AncientDbReplacer(); err != nil { + log.Error("Failed to rename new ancient directory") + return err + } + log.Info("Block prune successfully") + return nil + } + name := "chaindata" + if err := blockpruner.BlockPruneBackUp(name, c.cache, fdHandles, "", false, false); err != nil { + log.Error("Failed to back up block", "err", err) + return err + } + + log.Info("backup block successfully") + + // After backing up successfully, rename the new ancientdb name to the original one, and delete the old ancientdb + if err := blockpruner.AncientDbReplacer(); err != nil { + return err + } + + lock.Release() + log.Info("Block prune successfully") + + return nil +} From 8bd357f08d2e3c92d1a48f42d304b41039617ef2 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 20 Feb 2023 09:54:53 +0800 Subject: [PATCH 07/53] fix typo Signed-off-by: Delweng --- core/state/pruner/pruner.go | 20 ++++++++++---------- internal/cli/snapshot.go | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 0489c57c91..2c24770567 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -356,11 +356,11 @@ func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientP } } -func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace string, readonly, interrupt bool) error { +func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace string, readonly, interrupt bool) error { // Open old db wrapper. chainDb, err := p.node.OpenDatabaseWithFreezer(name, cache, handles, p.oldAncientPath, namespace, readonly, true, interrupt) if err != nil { - log.Error("Failed to open ancient database", "err=", err) + log.Error("Failed to open ancient database", "err", err) return err } defer chainDb.Close() @@ -449,19 +449,19 @@ func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace str } } lock.Release() - log.Info("block back up done", "current start blockNumber in ancientDB", startBlockNumber) + log.Info("block backup done", "current start blockNumber in ancientDB", startBlockNumber) return nil } -// Backup the ancient data for the old ancient db, i.e. the most recent 128 blocks in ancient db. -func (p *BlockPruner) BlockPruneBackUp(name string, cache, handles int, namespace string, readonly, interrupt bool) error { +// BlockPruneBackup backup the ancient data for the old ancient db, i.e. the most recent 128 blocks in ancient db. +func (p *BlockPruner) BlockPruneBackup(name string, cache, handles int, namespace string, readonly, interrupt bool) error { start := time.Now() - if err := p.backUpOldDb(name, cache, handles, namespace, readonly, interrupt); err != nil { + if err := p.backupOldDb(name, cache, handles, namespace, readonly, interrupt); err != nil { return err } - log.Info("Block pruning BackUp successfully", "time duration since start is", common.PrettyDuration(time.Since(start))) + log.Info("Block pruning backup successfully", "time duration since start is", common.PrettyDuration(time.Since(start))) return nil } @@ -489,13 +489,13 @@ func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, names } if flockOfAncientBack { // Indicating the oldOffset/newOffset have already been updated. - if err := p.BlockPruneBackUp(name, cache, handles, namespace, readonly, true); err != nil { + if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, true); err != nil { log.Error("Failed to prune") return err } } else { // Indicating the flock did not exist and the new offset did not be updated, so just handle this case as usual. - if err := p.BlockPruneBackUp(name, cache, handles, namespace, readonly, false); err != nil { + if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, false); err != nil { log.Error("Failed to prune") return err } @@ -509,7 +509,7 @@ func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, names log.Info("New ancientDB_backup did not exist in interruption scenario") // Indicating new ancientDB even did not be created, just prune starting at backup from startBlockNumber as usual, // in this case, the new offset have not been written into kvDB. - if err := p.BlockPruneBackUp(name, cache, handles, namespace, readonly, false); err != nil { + if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, false); err != nil { log.Error("Failed to prune") return err } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 8e4bf871ef..9c44eaad46 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -441,14 +441,14 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, chaindb ethdb.Database, return nil } name := "chaindata" - if err := blockpruner.BlockPruneBackUp(name, c.cache, fdHandles, "", false, false); err != nil { - log.Error("Failed to back up block", "err", err) + if err := blockpruner.BlockPruneBackup(name, c.cache, fdHandles, "", false, false); err != nil { + log.Error("Failed to backup block", "err", err) return err } log.Info("backup block successfully") - // After backing up successfully, rename the new ancientdb name to the original one, and delete the old ancientdb + // After backup successfully, rename the new ancientdb name to the original one, and delete the old ancientdb if err := blockpruner.AncientDbReplacer(); err != nil { return err } From b34d1a5a34d163299e302d38d5b9fb467f9c9af4 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 20 Feb 2023 10:24:47 +0800 Subject: [PATCH 08/53] cli/snapshot: fix the issue of dup open db error Signed-off-by: Delweng --- core/state/pruner/pruner.go | 4 +--- internal/cli/snapshot.go | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 2c24770567..cc09ad13e2 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -339,16 +339,14 @@ func (p *Pruner) Prune(root common.Hash) error { } type BlockPruner struct { - db ethdb.Database oldAncientPath string newAncientPath string node *node.Node BlockAmountReserved uint64 } -func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientPath string, BlockAmountReserved uint64) *BlockPruner { +func NewBlockPruner(n *node.Node, oldAncientPath, newAncientPath string, BlockAmountReserved uint64) *BlockPruner { return &BlockPruner{ - db: db, oldAncientPath: oldAncientPath, newAncientPath: newAncientPath, node: n, diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 9c44eaad46..695bd3ab81 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -13,7 +13,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/cli/flagset" "github.com/ethereum/go-ethereum/internal/cli/server" "github.com/ethereum/go-ethereum/log" @@ -297,20 +296,13 @@ func (c *PruneBlockCommand) Run(args []string) int { return 1 } - chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, c.cache, dbHandles, c.datadirAncient, "", false, true, false) - if err != nil { - c.UI.Error(err.Error()) - return 1 - } - defer chaindb.Close() - - err = c.accessDb(node, chaindb) + err = c.accessDb(node, dbHandles) if err != nil { c.UI.Error(err.Error()) return 1 } - err = c.pruneBlock(node, chaindb, dbHandles) + err = c.pruneBlock(node, dbHandles) if err != nil { c.UI.Error(err.Error()) return 1 @@ -318,7 +310,12 @@ func (c *PruneBlockCommand) Run(args []string) int { return 0 } -func (c *PruneBlockCommand) accessDb(stack *node.Node, chaindb ethdb.Database) error { +func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { + chaindb, err := stack.OpenDatabaseWithFreezer(chaindataPath, c.cache, dbHandles, c.datadirAncient, "", false, true, false) + if err != nil { + return err + } + defer chaindb.Close() if !c.checkSnapshotWithMPT { return nil } @@ -400,7 +397,7 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, chaindb ethdb.Database) e return nil } -func (c *PruneBlockCommand) pruneBlock(stack *node.Node, chaindb ethdb.Database, fdHandles int) error { +func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { oldAncientPath := c.datadirAncient if !filepath.IsAbs(oldAncientPath) { oldAncientPath = stack.ResolvePath(oldAncientPath) @@ -412,7 +409,7 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, chaindb ethdb.Database, } newAncientPath := filepath.Join(path, "ancient_back") - blockpruner := pruner.NewBlockPruner(chaindb, stack, oldAncientPath, newAncientPath, c.blockAmountReserved) + blockpruner := pruner.NewBlockPruner(stack, oldAncientPath, newAncientPath, c.blockAmountReserved) lock, exist, err := fileutil.Flock(filepath.Join(oldAncientPath, "PRUNEFLOCK")) if err != nil { @@ -446,7 +443,7 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, chaindb ethdb.Database, return err } - log.Info("backup block successfully") + log.Info("Block backup successfully") // After backup successfully, rename the new ancientdb name to the original one, and delete the old ancientdb if err := blockpruner.AncientDbReplacer(); err != nil { From b5f9741b33dbc6f9302b79cba238eb3de9d45851 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 20 Feb 2023 20:20:42 +0800 Subject: [PATCH 09/53] cli/snapshot: resolve datadir and ancient before backup Signed-off-by: Delweng --- internal/cli/snapshot.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 695bd3ab81..404abb6a94 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -313,9 +313,10 @@ func (c *PruneBlockCommand) Run(args []string) int { func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { chaindb, err := stack.OpenDatabaseWithFreezer(chaindataPath, c.cache, dbHandles, c.datadirAncient, "", false, true, false) if err != nil { - return err + return fmt.Errorf("failed to accessdb %v", err) } defer chaindb.Close() + if !c.checkSnapshotWithMPT { return nil } @@ -398,8 +399,13 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { } func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { + name := "chaindata" + oldAncientPath := c.datadirAncient - if !filepath.IsAbs(oldAncientPath) { + switch { + case oldAncientPath == "": + oldAncientPath = filepath.Join(stack.ResolvePath(name), "ancient") + case !filepath.IsAbs(oldAncientPath): oldAncientPath = stack.ResolvePath(oldAncientPath) } @@ -437,7 +443,6 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { log.Info("Block prune successfully") return nil } - name := "chaindata" if err := blockpruner.BlockPruneBackup(name, c.cache, fdHandles, "", false, false); err != nil { log.Error("Failed to backup block", "err", err) return err From b4ad6c0c9529d3cc938d6616872998c1ea7e1630 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 20 Feb 2023 20:20:55 +0800 Subject: [PATCH 10/53] core: more prune-block log Signed-off-by: Delweng --- core/rawdb/freezer.go | 2 +- core/state/pruner/pruner.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index a92ae5a248..d8451ac456 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -185,7 +185,7 @@ func newFreezer(datadir string, namespace string, readonly bool, offset uint64, // Create the write batch. freezer.writeBatch = newFreezerBatch(freezer) - log.Info("Opened ancient database", "database", datadir, "readonly", readonly) + log.Info("Opened ancient database", "database", datadir, "readonly", readonly, "frozen", freezer.frozen, "offset", freezer.offset) return freezer, nil } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index cc09ad13e2..6229991fc2 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -355,6 +355,7 @@ func NewBlockPruner(n *node.Node, oldAncientPath, newAncientPath string, BlockAm } func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace string, readonly, interrupt bool) error { + log.Info("backup", "oldAncientPath", p.oldAncientPath, "newAncientPath", p.newAncientPath) // Open old db wrapper. chainDb, err := p.node.OpenDatabaseWithFreezer(name, cache, handles, p.oldAncientPath, namespace, readonly, true, interrupt) if err != nil { From cf25261502957340651527eaac114020bdba1f5e Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 20 Feb 2023 20:50:46 +0800 Subject: [PATCH 11/53] core: truncatetail missing f.offset Signed-off-by: Delweng --- core/rawdb/freezer.go | 2 +- core/state/pruner/pruner.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index d8451ac456..b08db8eace 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -330,7 +330,7 @@ func (f *freezer) TruncateHead(items uint64) error { return nil } for _, table := range f.tables { - if err := table.truncateHead(items); err != nil { + if err := table.truncateHead(items - f.offset); err != nil { return err } } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 6229991fc2..a6a6752207 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -401,7 +401,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // For every round, newoffset actually equals to the startBlockNumber in ancient backup db. frdbBack, err := rawdb.NewFreezerDb(chainDb, p.newAncientPath, namespace, readonly, startBlockNumber) if err != nil { - log.Error("Failed to create ancient freezer backup", "err=", err) + log.Error("Failed to create ancient freezer backup", "err", err) return err } defer frdbBack.Close() @@ -438,7 +438,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str } // Write into new ancient_back db. if _, err := rawdb.WriteAncientBlocks(frdbBack, []*types.Block{block}, []types.Receipts{receipts}, []types.Receipts{borReceipts}, td); err != nil { - log.Error("failed to write new ancient", "error", err) + log.Error("failed to write new ancient", "err", err) return err } // Print the log every 5s for better trace. From 7397e5acc2bb4966cdebd7c788f0d0172c52c601 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 20 Feb 2023 21:00:38 +0800 Subject: [PATCH 12/53] core/rawdb: indextx adjust offset of pruned block Signed-off-by: Delweng --- core/rawdb/chain_iterator.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 6c79a57aee..c0b0a95101 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -179,6 +179,10 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool // There is a passed channel, the whole procedure will be interrupted if any // signal received. func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { + // adjust range boundary for pruned block + if offset := db.AncientOffSet(); offset > from { + from = offset + } // short circuit for invalid range if from >= to { return @@ -271,6 +275,10 @@ func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, inte // There is a passed channel, the whole procedure will be interrupted if any // signal received. func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { + // adjust range boundary for pruned block + if offset := db.AncientOffSet(); offset > from { + from = offset + } // short circuit for invalid range if from >= to { return From f7833b73662a507769ff26a22912d33ed91fc741 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 21 Feb 2023 12:06:27 +0800 Subject: [PATCH 13/53] core/rawdb: freezer batch should implement the offset commit, ref https://github.com/bnb-chain/bsc/pull/1005 Signed-off-by: Delweng --- core/rawdb/freezer.go | 2 +- core/rawdb/freezer_batch.go | 13 ++++++++----- core/rawdb/freezer_table.go | 2 +- core/rawdb/freezer_table_test.go | 26 +++++++++++++------------- core/rawdb/freezer_test.go | 4 ++-- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index b08db8eace..29f4416f49 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -712,7 +712,7 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { return err } var ( - batch = newTable.newBatch() + batch = newTable.newBatch(f.offset) out []byte start = time.Now() logged = time.Now() diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 864a7f5e98..044bad7da1 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -37,7 +37,7 @@ type freezerBatch struct { func newFreezerBatch(f *freezer) *freezerBatch { batch := &freezerBatch{tables: make(map[string]*freezerTableBatch, len(f.tables))} for kind, table := range f.tables { - batch.tables[kind] = table.newBatch() + batch.tables[kind] = table.newBatch(f.offset) } return batch } @@ -91,11 +91,12 @@ type freezerTableBatch struct { indexBuffer []byte curItem uint64 // expected index of next append totalBytes int64 // counts written bytes since reset + offset uint64 } // newBatch creates a new batch for the freezer table. -func (t *freezerTable) newBatch() *freezerTableBatch { - batch := &freezerTableBatch{t: t} +func (t *freezerTable) newBatch(offset uint64) *freezerTableBatch { + batch := &freezerTableBatch{t: t, offset: offset} if !t.noCompression { batch.sb = new(snappyBuffer) } @@ -107,7 +108,8 @@ func (t *freezerTable) newBatch() *freezerTableBatch { func (batch *freezerTableBatch) reset() { batch.dataBuffer = batch.dataBuffer[:0] batch.indexBuffer = batch.indexBuffer[:0] - batch.curItem = atomic.LoadUint64(&batch.t.items) + curItem := batch.t.items + batch.offset + batch.curItem = atomic.LoadUint64(&curItem) batch.totalBytes = 0 } @@ -201,7 +203,8 @@ func (batch *freezerTableBatch) commit() error { // Update headBytes of table. batch.t.headBytes += dataSize - atomic.StoreUint64(&batch.t.items, batch.curItem) + items := batch.curItem - batch.offset + atomic.StoreUint64(&batch.t.items, items) // Update metrics. batch.t.sizeGauge.Inc(dataSize + indexSize) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index a076ded067..a01d8dfd9a 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -931,7 +931,7 @@ func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { // Fill adds empty data till given number (convenience method for backward compatibilty) func (t *freezerTable) Fill(number uint64) error { if t.items < number { - b := t.newBatch() + b := t.newBatch(0) log.Info("Filling all data into freezer for backward compatablity", "name", t.name, "items", t.items, "number", number) for t.items < number { if err := b.Append(t.items, nil); err != nil { diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0bddcf7211..5cbb140b04 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -99,7 +99,7 @@ func TestFreezerBasicsClosing(t *testing.T) { // In-between writes, the table is closed and re-opened. for x := 0; x < 255; x++ { data := getChunk(15, x) - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(uint64(x), data)) require.NoError(t, batch.commit()) f.Close() @@ -227,7 +227,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { t.Errorf("Expected error for missing index entry") } // We should now be able to store items again, from item = 1 - batch := f.newBatch() + batch := f.newBatch(0) for x := 1; x < 0xff; x++ { require.NoError(t, batch.AppendRaw(uint64(x), getChunk(15, ^x))) } @@ -417,7 +417,7 @@ func TestFreezerRepairFirstFile(t *testing.T) { t.Fatal(err) } // Write 80 bytes, splitting out into two files - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(0, getChunk(40, 0xFF))) require.NoError(t, batch.AppendRaw(1, getChunk(40, 0xEE))) require.NoError(t, batch.commit()) @@ -455,7 +455,7 @@ func TestFreezerRepairFirstFile(t *testing.T) { } // Write 40 bytes - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(1, getChunk(40, 0xDD))) require.NoError(t, batch.commit()) @@ -512,7 +512,7 @@ func TestFreezerReadAndTruncate(t *testing.T) { f.truncateHead(0) // Write the data again - batch := f.newBatch() + batch := f.newBatch(0) for x := 0; x < 30; x++ { require.NoError(t, batch.AppendRaw(uint64(x), getChunk(15, ^x))) } @@ -534,7 +534,7 @@ func TestFreezerOffset(t *testing.T) { } // Write 6 x 20 bytes, splitting out into three files - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) @@ -598,7 +598,7 @@ func TestFreezerOffset(t *testing.T) { t.Log(f.dumpIndexString(0, 100)) // It should allow writing item 6. - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99))) require.NoError(t, batch.commit()) @@ -676,7 +676,7 @@ func TestTruncateTail(t *testing.T) { } // Write 7 x 20 bytes, splitting out into four files - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) @@ -791,7 +791,7 @@ func TestTruncateHead(t *testing.T) { } // Write 7 x 20 bytes, splitting out into four files - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) @@ -816,7 +816,7 @@ func TestTruncateHead(t *testing.T) { }) // Append new items - batch = f.newBatch() + batch = f.newBatch(0) require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x11))) @@ -880,7 +880,7 @@ func getChunk(size int, b int) []byte { func writeChunks(t *testing.T, ft *freezerTable, n int, length int) { t.Helper() - batch := ft.newBatch() + batch := ft.newBatch(0) for i := 0; i < n; i++ { if err := batch.AppendRaw(uint64(i), getChunk(length, i)); err != nil { t.Fatalf("AppendRaw(%d, ...) returned error: %v", i, err) @@ -1076,7 +1076,7 @@ func TestFreezerReadonly(t *testing.T) { // Case 5: Now write some data via a batch. // This should fail either during AppendRaw or Commit - batch := f.newBatch() + batch := f.newBatch(0) writeErr := batch.AppendRaw(32, make([]byte, 1)) if writeErr == nil { writeErr = batch.commit() @@ -1231,7 +1231,7 @@ func runRandTest(rt randTest) bool { } case opAppend: - batch := f.newBatch() + batch := f.newBatch(0) for i := 0; i < len(step.items); i++ { batch.AppendRaw(step.items[i], step.blobs[i]) } diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index 3379736047..e8857042fb 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -268,12 +268,12 @@ func TestFreezerReadonlyValidate(t *testing.T) { t.Fatal("can't open freezer", err) } var item = make([]byte, 1024) - aBatch := f.tables["a"].newBatch() + aBatch := f.tables["a"].newBatch(0) require.NoError(t, aBatch.AppendRaw(0, item)) require.NoError(t, aBatch.AppendRaw(1, item)) require.NoError(t, aBatch.AppendRaw(2, item)) require.NoError(t, aBatch.commit()) - bBatch := f.tables["b"].newBatch() + bBatch := f.tables["b"].newBatch(0) require.NoError(t, bBatch.AppendRaw(0, item)) require.NoError(t, bBatch.commit()) if f.tables["a"].items != 3 { From 786c95c82bd4e3f392e0eddd8b75cca9d352e173 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 21 Feb 2023 16:08:03 +0800 Subject: [PATCH 14/53] core: check of ancientdb, backport https://github.com/bnb-chain/bsc/pull/817 Signed-off-by: Delweng --- core/headerchain.go | 3 +++ core/rawdb/chain_iterator.go | 5 ++++- eth/state_accessor.go | 21 ++++++++++++--------- internal/ethapi/api.go | 3 +++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/core/headerchain.go b/core/headerchain.go index 83e1125305..aa857812f1 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -168,6 +168,9 @@ func (hc *HeaderChain) Reorg(headers []*types.Header) error { headHash = header.Hash() ) for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash { + if frozen, _ := hc.chainDb.Ancients(); frozen == headNumber { + break + } rawdb.WriteCanonicalHash(batch, headHash, headNumber) if headNumber == 0 { break // It shouldn't be reached diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index c0b0a95101..931e1cb87f 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -99,7 +99,10 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool number uint64 rlp rlp.RawValue } - if to == from { + if offset := db.AncientOffSet(); offset > from { + from = offset + } + if to <= from { return nil } threads := to - from diff --git a/eth/state_accessor.go b/eth/state_accessor.go index f01db93a67..daaccac747 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -36,15 +36,15 @@ import ( // base layer statedb can be passed then it's regarded as the statedb of the // parent block. // Parameters: -// - block: The block for which we want the state (== state at the stateRoot of the parent) -// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state -// - base: If the caller is tracing multiple blocks, the caller can provide the parent state -// continuously from the callsite. -// - checklive: if true, then the live 'blockchain' state database is used. If the caller want to -// perform Commit or other 'save-to-disk' changes, this should be set to false to avoid -// storing trash persistently -// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, -// it would be preferrable to start from a fresh state, if we have it on disk. +// - block: The block for which we want the state (== state at the stateRoot of the parent) +// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state +// - base: If the caller is tracing multiple blocks, the caller can provide the parent state +// continuously from the callsite. +// - checklive: if true, then the live 'blockchain' state database is used. If the caller want to +// perform Commit or other 'save-to-disk' changes, this should be set to false to avoid +// storing trash persistently +// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, +// it would be preferrable to start from a fresh state, if we have it on disk. func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { var ( current *types.Block @@ -72,6 +72,9 @@ func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state // The optional base statedb is given, mark the start point as parent block statedb, database, report = base, base.Database(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if current == nil { + return nil, fmt.Errorf("missing parent block %v %d", block.ParentHash(), block.NumberU64()-1) + } } else { // Otherwise try to reexec blocks until we find a state or reach our limit current = block diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9f6aed5b96..95c5bf9f5d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -636,6 +636,9 @@ func (s *PublicBlockChainAPI) GetTransactionReceiptsByBlock(ctx context.Context, if err != nil { return nil, err } + if receipts == nil { + return nil, fmt.Errorf("block %d receipts not found", block.NumberU64()) + } txs := block.Transactions() From ec275a964d23091774c71e00303d1c59603f553f Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 23 Feb 2023 20:59:36 +0800 Subject: [PATCH 15/53] core/state: read raw borReceipt to backup Signed-off-by: Delweng --- core/state/pruner/pruner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a6a6752207..a278342d54 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -429,7 +429,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str blockHash := rawdb.ReadCanonicalHash(chainDb, blockNumber) block := rawdb.ReadBlock(chainDb, blockHash, blockNumber) receipts := rawdb.ReadRawReceipts(chainDb, blockHash, blockNumber) - borReceipts := []*types.Receipt{rawdb.ReadBorReceipt(chainDb, blockHash, blockNumber)} + borReceipts := []*types.Receipt{rawdb.ReadRawBorReceipt(chainDb, blockHash, blockNumber)} // Calculate the total difficulty of the block td := rawdb.ReadTd(chainDb, blockHash, blockNumber) @@ -442,7 +442,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str return err } // Print the log every 5s for better trace. - if common.PrettyDuration(time.Since(start)) > common.PrettyDuration(5*time.Second) { + if time.Since(start) > 5*time.Second { log.Info("block backup process running successfully", "current blockNumber for backup", blockNumber) start = time.Now() } From afdeeec45098414df9feb9ea7e1419eaee2f7f8c Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 11:54:12 +0800 Subject: [PATCH 16/53] core/rawdb: bor receipt maybe in []Receipt or Receipt RLP format Signed-off-by: Delweng --- core/rawdb/bor_receipt.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/rawdb/bor_receipt.go b/core/rawdb/bor_receipt.go index e225083741..33e75fea57 100644 --- a/core/rawdb/bor_receipt.go +++ b/core/rawdb/bor_receipt.go @@ -91,8 +91,16 @@ func ReadRawBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types. // Convert the receipts from their storage form to their internal representation var storageReceipt types.ReceiptForStorage if err := rlp.DecodeBytes(data, &storageReceipt); err != nil { - log.Error("Invalid receipt array RLP", "hash", hash, "err", err) - return nil + storageReceipts := []*types.ReceiptForStorage{} + if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { + log.Error("Invalid receipt array RLP", "hash", hash, "err", err) + return nil + } + if len(storageReceipts) != 1 { + log.Error("Invalid bor receipt array RLP", "hash", hash, "err", err) + return nil + } + return (*types.Receipt)(storageReceipts[0]) } return (*types.Receipt)(&storageReceipt) From f91030b29d3d826dc804af2d42227d09ed15c237 Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 17:02:13 +0800 Subject: [PATCH 17/53] core/state: typo and error msg Signed-off-by: Delweng --- core/rawdb/schema.go | 2 +- core/state/pruner/pruner.go | 81 +++++++++++++++---------------------- internal/cli/snapshot.go | 4 +- 3 files changed, 35 insertions(+), 52 deletions(-) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index cc9ef0e534..1765bfe2e4 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -72,7 +72,7 @@ var ( // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") - // offSet of new updated ancientDB. + // offSet of the new updated ancientDB. offSetOfCurrentAncientFreezer = []byte("offSetOfCurrentAncientFreezer") // offSet of the ancientDB before updated version. diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a278342d54..509ac82c19 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -355,60 +355,56 @@ func NewBlockPruner(n *node.Node, oldAncientPath, newAncientPath string, BlockAm } func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace string, readonly, interrupt bool) error { - log.Info("backup", "oldAncientPath", p.oldAncientPath, "newAncientPath", p.newAncientPath) + log.Info("Backup old ancientDB", "oldAncientPath", p.oldAncientPath, "newAncientPath", p.newAncientPath) // Open old db wrapper. chainDb, err := p.node.OpenDatabaseWithFreezer(name, cache, handles, p.oldAncientPath, namespace, readonly, true, interrupt) if err != nil { - log.Error("Failed to open ancient database", "err", err) - return err + return fmt.Errorf("Failed to open ancient database %v", err) } defer chainDb.Close() - log.Info("chainDB opened successfully") // Get the number of items in old ancient db. itemsOfAncient, err := chainDb.ItemAmountInAncient() - log.Info("the number of items in ancientDB is ", "itemsOfAncient", itemsOfAncient) + log.Info("ChainDB opened successfully", "itemsOfAncient", itemsOfAncient) // If we can't access the freezer or it's empty, abort. if err != nil || itemsOfAncient == 0 { - log.Error("can't access the freezer or it's empty, abort") return errors.New("can't access the freezer or it's empty, abort") } // If the items in freezer is less than the block amount that we want to reserve, it is not enough, should stop. if itemsOfAncient < p.BlockAmountReserved { - log.Error("the number of old blocks is not enough to reserve,", "ancient items", itemsOfAncient, "the amount specified", p.BlockAmountReserved) - return errors.New("the number of old blocks is not enough to reserve") + return fmt.Errorf("the number of old blocks is not enough to reserve, ancientItems=%d, specifiedReservedBlockAmount=%d", itemsOfAncient, p.BlockAmountReserved) + } else if itemsOfAncient == p.BlockAmountReserved { + return fmt.Errorf("the number of old blocks is the same to reserved blocks, ancientItems=%d", itemsOfAncient) } - var oldOffSet uint64 + var oldOffset uint64 if interrupt { // The interrupt scecario within this function is specific for old and new ancientDB exsisted concurrently, // should use last version of offset for oldAncientDB, because current offset is // actually of the new ancientDB_Backup, but what we want is the offset of ancientDB being backup. - oldOffSet = rawdb.ReadOffSetOfLastAncientFreezer(chainDb) + oldOffset = rawdb.ReadOffSetOfLastAncientFreezer(chainDb) } else { // Using current version of ancientDB for oldOffSet because the db for backup is current version. - oldOffSet = rawdb.ReadOffSetOfCurrentAncientFreezer(chainDb) + oldOffset = rawdb.ReadOffSetOfCurrentAncientFreezer(chainDb) } - log.Info("the oldOffSet is ", "oldOffSet", oldOffSet) // Get the start BlockNumber for pruning. - startBlockNumber := oldOffSet + itemsOfAncient - p.BlockAmountReserved - log.Info("new offset/new startBlockNumber is ", "new offset", startBlockNumber) + startBlockNumber := oldOffset + itemsOfAncient - p.BlockAmountReserved + log.Info("Prune info", "oldOffset", oldOffset, "newOffset", startBlockNumber) - // Create new ancientdb backup and record the new and last version of offset in kvDB as well. + // Create new ancientDB backup and record the new and last version of offset in kvDB as well. // For every round, newoffset actually equals to the startBlockNumber in ancient backup db. frdbBack, err := rawdb.NewFreezerDb(chainDb, p.newAncientPath, namespace, readonly, startBlockNumber) if err != nil { - log.Error("Failed to create ancient freezer backup", "err", err) - return err + return fmt.Errorf("Failed to create ancient freezer backup: %v", err) } defer frdbBack.Close() offsetBatch := chainDb.NewBatch() rawdb.WriteOffSetOfCurrentAncientFreezer(offsetBatch, startBlockNumber) - rawdb.WriteOffSetOfLastAncientFreezer(offsetBatch, oldOffSet) + rawdb.WriteOffSetOfLastAncientFreezer(offsetBatch, oldOffset) if err := offsetBatch.Write(); err != nil { log.Crit("Failed to write offset into disk", "err", err) } @@ -416,16 +412,15 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // It's guaranteed that the old/new offsets are updated as well as the new ancientDB are created if this flock exist. lock, _, err := fileutil.Flock(filepath.Join(p.newAncientPath, "PRUNEFLOCKBACK")) if err != nil { - log.Error("file lock error", "err", err) - return err + return fmt.Errorf("file lock error: %v", err) } - log.Info("prune info", "old offset", oldOffSet, "number of items in ancientDB", itemsOfAncient, "amount to reserve", p.BlockAmountReserved) - log.Info("new offset/new startBlockNumber recorded successfully ", "new offset", startBlockNumber) + log.Info("Prune info", "old offset", oldOffset, "number of items in ancientDB", itemsOfAncient, "number of blocks to reserve", p.BlockAmountReserved) + log.Info("Record newOffset/newStartBlockNumber successfully", "newOffset", startBlockNumber) start := time.Now() // All ancient data after and including startBlockNumber should write into new ancientDB ancient_back. - for blockNumber := startBlockNumber; blockNumber < itemsOfAncient+oldOffSet; blockNumber++ { + for blockNumber := startBlockNumber; blockNumber < itemsOfAncient+oldOffset; blockNumber++ { blockHash := rawdb.ReadCanonicalHash(chainDb, blockNumber) block := rawdb.ReadBlock(chainDb, blockHash, blockNumber) receipts := rawdb.ReadRawReceipts(chainDb, blockHash, blockNumber) @@ -438,17 +433,16 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str } // Write into new ancient_back db. if _, err := rawdb.WriteAncientBlocks(frdbBack, []*types.Block{block}, []types.Receipts{receipts}, []types.Receipts{borReceipts}, td); err != nil { - log.Error("failed to write new ancient", "err", err) - return err + return fmt.Errorf("failed to write new ancient error: %v", err) } // Print the log every 5s for better trace. if time.Since(start) > 5*time.Second { - log.Info("block backup process running successfully", "current blockNumber for backup", blockNumber) + log.Info("Block backup process running successfully", "current blockNumber for backup", blockNumber) start = time.Now() } } lock.Release() - log.Info("block backup done", "current start blockNumber in ancientDB", startBlockNumber) + log.Info("Backup old ancientDB done", "current start blockNumber in ancientDB", startBlockNumber) return nil } @@ -457,6 +451,7 @@ func (p *BlockPruner) BlockPruneBackup(name string, cache, handles int, namespac start := time.Now() if err := p.backupOldDb(name, cache, handles, namespace, readonly, interrupt); err != nil { + log.Error("Backup old ancientDB error", "err", err) return err } @@ -466,42 +461,36 @@ func (p *BlockPruner) BlockPruneBackup(name string, cache, handles int, namespac func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, namespace string, readonly bool) error { log.Info("RecoverInterruption for block prune") - newExist, err := CheckFileExist(p.newAncientPath) + newExist, err := checkFileExist(p.newAncientPath) if err != nil { - log.Error("newAncientDb path error") - return err + return fmt.Errorf("newAncientDB path error %v", err) } if newExist { log.Info("New ancientDB_backup existed in interruption scenario") - flockOfAncientBack, err := CheckFileExist(filepath.Join(p.newAncientPath, "PRUNEFLOCKBACK")) + flockOfAncientBack, err := checkFileExist(filepath.Join(p.newAncientPath, "PRUNEFLOCKBACK")) if err != nil { - log.Error("Failed to check flock of ancientDB_Back %v", err) - return err + return fmt.Errorf("failed to check flock of ancientDB_Back %v", err) } // Indicating both old and new ancientDB existed concurrently. - // Delete directly for the new ancientdb to prune from start, e.g.: path ../chaindb/ancient_backup + // Delete directly for the new ancientDB to prune from start, e.g.: path ../chaindb/ancient_backup if err := os.RemoveAll(p.newAncientPath); err != nil { - log.Error("Failed to remove old ancient directory %v", err) - return err + return fmt.Errorf("failed to remove old ancient directory %v", err) } if flockOfAncientBack { // Indicating the oldOffset/newOffset have already been updated. if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, true); err != nil { - log.Error("Failed to prune") return err } } else { // Indicating the flock did not exist and the new offset did not be updated, so just handle this case as usual. if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, false); err != nil { - log.Error("Failed to prune") return err } } if err := p.AncientDbReplacer(); err != nil { - log.Error("Failed to replace ancientDB") return err } } else { @@ -509,11 +498,9 @@ func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, names // Indicating new ancientDB even did not be created, just prune starting at backup from startBlockNumber as usual, // in this case, the new offset have not been written into kvDB. if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, false); err != nil { - log.Error("Failed to prune") return err } if err := p.AncientDbReplacer(); err != nil { - log.Error("Failed to replace ancientDB") return err } } @@ -521,7 +508,7 @@ func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, names return nil } -func CheckFileExist(path string) (bool, error) { +func checkFileExist(path string) (bool, error) { if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { // Indicating the file didn't exist. @@ -533,16 +520,14 @@ func CheckFileExist(path string) (bool, error) { } func (p *BlockPruner) AncientDbReplacer() error { - // Delete directly for the old ancientdb, e.g.: path ../chaindb/ancient + // Delete directly for the old ancientDB, e.g.: path ../chaindb/ancient if err := os.RemoveAll(p.oldAncientPath); err != nil { - log.Error("Failed to remove old ancient directory %v", err) - return err + return fmt.Errorf("failed to remove old ancient directory %v", err) } - // Rename the new ancientdb path same to the old + // Rename the new ancientDB path same to the old if err := os.Rename(p.newAncientPath, p.oldAncientPath); err != nil { - log.Error("Failed to rename new ancient directory") - return err + return fmt.Errorf("failed to rename new ancient directory %v", err) } return nil } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 404abb6a94..6f8753a0c3 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -424,7 +424,7 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { } if exist { defer lock.Release() - log.Info("file lock existed, waiting for prune recovery and continue", "err", err) + log.Info("File lock existed, waiting for prune recovery and continue", "err", err) if err := blockpruner.RecoverInterruption("chaindata", c.cache, fdHandles, "", false); err != nil { log.Error("Pruning failed", "err", err) return err @@ -437,14 +437,12 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { // No file lock found for old ancientDB but new ancientDB exsisted, indicating the geth was interrupted // after old ancientDB removal, this happened after backup successfully, so just rename the new ancientDB if err := blockpruner.AncientDbReplacer(); err != nil { - log.Error("Failed to rename new ancient directory") return err } log.Info("Block prune successfully") return nil } if err := blockpruner.BlockPruneBackup(name, c.cache, fdHandles, "", false, false); err != nil { - log.Error("Failed to backup block", "err", err) return err } From 727656cb49f3b73a1156df52851215807f9df3fe Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 17:07:05 +0800 Subject: [PATCH 18/53] core/rawdb: offSet -> offset Signed-off-by: Delweng --- core/rawdb/database.go | 27 ++++++++++++++++----------- core/rawdb/schema.go | 8 ++++---- core/state/pruner/pruner.go | 8 ++++---- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index ce4f1dce3d..842c87efbd 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -167,29 +167,34 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } -func ReadOffSetOfCurrentAncientFreezer(db ethdb.KeyValueReader) uint64 { - offset, _ := db.Get(offSetOfCurrentAncientFreezer) +// ReadOffsetOfCurrentAncientFreezer reads the offset of current ancient freezer +func ReadOffsetOfCurrentAncientFreezer(db ethdb.KeyValueReader) uint64 { + offset, _ := db.Get(offsetOfCurrentAncientFreezer) if offset == nil { return 0 } return new(big.Int).SetBytes(offset).Uint64() } -func ReadOffSetOfLastAncientFreezer(db ethdb.KeyValueReader) uint64 { - offset, _ := db.Get(offSetOfLastAncientFreezer) +// ReadOffsetOfLastAncientFreezer reads the offset of last pruned ancient freezer +func ReadOffsetOfLastAncientFreezer(db ethdb.KeyValueReader) uint64 { + offset, _ := db.Get(offsetOfLastAncientFreezer) if offset == nil { return 0 } return new(big.Int).SetBytes(offset).Uint64() } -func WriteOffSetOfCurrentAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { - if err := db.Put(offSetOfCurrentAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { +// WriteOffsetOfCurrentAncientFreezer writes current offset of ancient freezer into ethdb +func WriteOffsetOfCurrentAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { + if err := db.Put(offsetOfCurrentAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { log.Crit("Failed to store offSetOfAncientFreezer", "err", err) } } -func WriteOffSetOfLastAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { - if err := db.Put(offSetOfLastAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { + +// WriteOffsetOfLastAncientFreezer writes the last offset of ancient freezer into ethdb +func WriteOffsetOfLastAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { + if err := db.Put(offsetOfLastAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil { log.Crit("Failed to store offSetOfAncientFreezer", "err", err) } } @@ -211,9 +216,9 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st var offset uint64 // The offset of ancientDB should be handled differently in different scenarios. if isLastOffset { - offset = ReadOffSetOfLastAncientFreezer(db) + offset = ReadOffsetOfLastAncientFreezer(db) } else { - offset = ReadOffSetOfCurrentAncientFreezer(db) + offset = ReadOffsetOfCurrentAncientFreezer(db) } // Create the idle freezer instance @@ -369,7 +374,7 @@ func (s *stat) Count() string { } func AncientInspect(db ethdb.Database) error { - offset := counter(ReadOffSetOfCurrentAncientFreezer(db)) + offset := counter(ReadOffsetOfCurrentAncientFreezer(db)) // Get number of ancient rows inside the freezer. ancients := counter(0) if count, err := db.ItemAmountInAncient(); err != nil { diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 1765bfe2e4..2b26b4bd30 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -72,11 +72,11 @@ var ( // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") - // offSet of the new updated ancientDB. - offSetOfCurrentAncientFreezer = []byte("offSetOfCurrentAncientFreezer") + // offset of the new updated ancientDB. + offsetOfCurrentAncientFreezer = []byte("offSetOfCurrentAncientFreezer") - // offSet of the ancientDB before updated version. - offSetOfLastAncientFreezer = []byte("offSetOfLastAncientFreezer") + // offset of the ancientDB before updated version. + offsetOfLastAncientFreezer = []byte("offSetOfLastAncientFreezer") // badBlockKey tracks the list of bad blocks seen by local badBlockKey = []byte("InvalidBlock") diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 509ac82c19..f6028d331a 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -384,10 +384,10 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // The interrupt scecario within this function is specific for old and new ancientDB exsisted concurrently, // should use last version of offset for oldAncientDB, because current offset is // actually of the new ancientDB_Backup, but what we want is the offset of ancientDB being backup. - oldOffset = rawdb.ReadOffSetOfLastAncientFreezer(chainDb) + oldOffset = rawdb.ReadOffsetOfLastAncientFreezer(chainDb) } else { // Using current version of ancientDB for oldOffSet because the db for backup is current version. - oldOffset = rawdb.ReadOffSetOfCurrentAncientFreezer(chainDb) + oldOffset = rawdb.ReadOffsetOfCurrentAncientFreezer(chainDb) } // Get the start BlockNumber for pruning. @@ -403,8 +403,8 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str defer frdbBack.Close() offsetBatch := chainDb.NewBatch() - rawdb.WriteOffSetOfCurrentAncientFreezer(offsetBatch, startBlockNumber) - rawdb.WriteOffSetOfLastAncientFreezer(offsetBatch, oldOffset) + rawdb.WriteOffsetOfCurrentAncientFreezer(offsetBatch, startBlockNumber) + rawdb.WriteOffsetOfLastAncientFreezer(offsetBatch, oldOffset) if err := offsetBatch.Write(); err != nil { log.Crit("Failed to write offset into disk", "err", err) } From 13292b0ca19f695d3102a7ecf225046ae67779e9 Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 17:13:24 +0800 Subject: [PATCH 19/53] cli/snapshot: comment Signed-off-by: Delweng --- internal/cli/snapshot.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 6f8753a0c3..f0b474a372 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -204,7 +204,7 @@ type PruneBlockCommand struct { func (c *PruneBlockCommand) MarkDown() string { items := []string{ "# Prune block", - "The ```bor snapshot prune-block``` command will prune ancient blockchain offline", + "The ```bor snapshot prune-block``` command will prune ancient blockchain data offline", c.Flags().MarkDown(), } @@ -220,7 +220,7 @@ func (c *PruneBlockCommand) Help() string { // Synopsis implements the cli.Command interface func (c *PruneBlockCommand) Synopsis() string { - return "Prune ancient data" + return "Prune ancient chaindata" } // Flags: datadir, datadir.ancient, cache.trie.journal, bloomfilter.size @@ -230,7 +230,7 @@ func (c *PruneBlockCommand) Flags() *flagset.Flagset { flags.StringFlag(&flagset.StringFlag{ Name: "datadir.ancient", Value: &c.datadirAncient, - Usage: "Path of the ancient data directory to store information", + Usage: "Path of the old ancient data directory", Default: "", }) @@ -243,7 +243,7 @@ func (c *PruneBlockCommand) Flags() *flagset.Flagset { }) flags.Uint64Flag(&flagset.Uint64Flag{ Name: "block-amount-reserved", - Usage: "Sets the expected remained amount of blocks for offline block prune", + Usage: "Sets the expected reserved number of blocks for offline block prune", Value: &c.blockAmountReserved, Default: 1024, }) @@ -258,7 +258,7 @@ func (c *PruneBlockCommand) Flags() *flagset.Flagset { flags.BoolFlag(&flagset.BoolFlag{ Name: "check-snapshot-with-mpt", Value: &c.checkSnapshotWithMPT, - Usage: "Path of the trie journal directory to store information", + Usage: "Enable checking between snapshot and MPT", }) return flags From 59e6752a636465fac447dd0e2c7de2c32cd10068 Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 21:30:08 +0800 Subject: [PATCH 20/53] cli/snapshot: add prune-block doc Signed-off-by: Delweng --- internal/cli/snapshot.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index f0b474a372..168c8b4609 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -34,6 +34,7 @@ func (a *SnapshotCommand) MarkDown() string { "# snapshot", "The ```snapshot``` command groups snapshot related actions:", "- [```snapshot prune-state```](./snapshot_prune-state.md): Prune state databases at the given datadir location.", + "- [```snapshot prune-block```](./snapshot_prune-block.md): Prune ancient chaindata at the given datadir location.", } return strings.Join(items, "\n\n") @@ -47,7 +48,11 @@ func (c *SnapshotCommand) Help() string { Prune the state trie: - $ bor snapshot prune-state` + $ bor snapshot prune-state + + Prune the ancient data: + + $ bor snapshot prune-block` } // Synopsis implements the cli.Command interface @@ -203,8 +208,16 @@ type PruneBlockCommand struct { // MarkDown implements cli.MarkDown interface func (c *PruneBlockCommand) MarkDown() string { items := []string{ - "# Prune block", - "The ```bor snapshot prune-block``` command will prune ancient blockchain data offline", + "# Prune ancient blockchain", + "The ```bor snapshot prune-block``` command will prune historical blockchain data stored in the ancientdb. The amount of blocks expected for remaining after prune can be specified via `block-amount-reserved` in this command, will prune and only remain the specified amount of old block data in ancientdb.", + ` +The brief workflow as below: + +1. backup the the number of specified number of blocks backward in original ancientdb into new ancient_backup, +2. then delete the original ancientdb dir and rename the ancient_backup to original one for replacement, +3. finally assemble the statedb and new ancientdb together. + +The purpose of doing it is because the block data will be moved into the ancient store when it becomes old enough(exceed the Threshold 90000), the disk usage will be very large over time, and is occupied mainly by ancientdb, so it's very necessary to do block data pruning, this feature will handle it.`, c.Flags().MarkDown(), } @@ -215,12 +228,12 @@ func (c *PruneBlockCommand) MarkDown() string { func (c *PruneBlockCommand) Help() string { return `Usage: bor snapshot prune-block - This command will prune blockchain databases at the given datadir location` + c.Flags().Help() + This command will prune ancient blockchain data at the given datadir location` + c.Flags().Help() } // Synopsis implements the cli.Command interface func (c *PruneBlockCommand) Synopsis() string { - return "Prune ancient chaindata" + return "Prune ancient blockchain data" } // Flags: datadir, datadir.ancient, cache.trie.journal, bloomfilter.size From 1b0abf04a5568802ab9b89d444925b0da55d821d Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 21:30:46 +0800 Subject: [PATCH 21/53] docs: add prune-block document Signed-off-by: Delweng --- docs/cli/README.md | 2 ++ docs/cli/snapshot.md | 4 +++- docs/cli/snapshot_prune-block.md | 30 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 docs/cli/snapshot_prune-block.md diff --git a/docs/cli/README.md b/docs/cli/README.md index d52a4fd836..d97e786686 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -46,6 +46,8 @@ - [```snapshot```](./snapshot.md) +- [```snapshot prune-block```](./snapshot_prune-block.md) + - [```snapshot prune-state```](./snapshot_prune-state.md) - [```status```](./status.md) diff --git a/docs/cli/snapshot.md b/docs/cli/snapshot.md index 376220749b..cc7a908088 100644 --- a/docs/cli/snapshot.md +++ b/docs/cli/snapshot.md @@ -2,4 +2,6 @@ The ```snapshot``` command groups snapshot related actions: -- [```snapshot prune-state```](./snapshot_prune-state.md): Prune state databases at the given datadir location. \ No newline at end of file +- [```snapshot prune-state```](./snapshot_prune-state.md): Prune state databases at the given datadir location. + +- [```snapshot prune-block```](./snapshot_prune-block.md): Prune ancient chaindata at the given datadir location. \ No newline at end of file diff --git a/docs/cli/snapshot_prune-block.md b/docs/cli/snapshot_prune-block.md new file mode 100644 index 0000000000..b966cb5e0f --- /dev/null +++ b/docs/cli/snapshot_prune-block.md @@ -0,0 +1,30 @@ +# Prune ancient blockchain + +The ```bor snapshot prune-block``` command will prune historical blockchain data stored in the ancientdb. The amount of blocks expected for remaining after prune can be specified via `block-amount-reserved` in this command, will prune and only remain the specified amount of old block data in ancientdb. + + +The brief workflow as below: + +1. backup the the number of specified number of blocks backward in original ancientdb into new ancient_backup, +2. then delete the original ancientdb dir and rename the ancient_backup to original one for replacement, +3. finally assemble the statedb and new ancientdb together. + +The purpose of doing it is because the block data will be moved into the ancient store when it becomes old enough(exceed the Threshold 90000), the disk usage will be very large over time, and is occupied mainly by ancientdb, so it's very necessary to do block data pruning, this feature will handle it. + +## Options + +- ```datadir```: Path of the data directory to store information + +- ```keystore```: Path of the data directory to store keys + +- ```datadir.ancient```: Path of the old ancient data directory + +- ```block-amount-reserved```: Sets the expected reserved number of blocks for offline block prune (default: 1024) + +- ```cache.triesinmemory```: Number of block states (tries) to keep in memory (default = 128) (default: 128) + +- ```check-snapshot-with-mpt```: Enable checking between snapshot and MPT (default: false) + +### Cache Options + +- ```cache```: Megabytes of memory allocated to internal caching (default: 1024) \ No newline at end of file From f9ae714dd4f522ae54bd899711766f4a002d3fd7 Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 1 Mar 2023 23:14:18 +0800 Subject: [PATCH 22/53] core/rawdb: print wrong bor-receipt length Signed-off-by: Delweng --- core/rawdb/bor_receipt.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rawdb/bor_receipt.go b/core/rawdb/bor_receipt.go index 33e75fea57..2da68e5356 100644 --- a/core/rawdb/bor_receipt.go +++ b/core/rawdb/bor_receipt.go @@ -93,11 +93,11 @@ func ReadRawBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types. if err := rlp.DecodeBytes(data, &storageReceipt); err != nil { storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { - log.Error("Invalid receipt array RLP", "hash", hash, "err", err) + log.Error("Invalid bor receipt array RLP", "number", number, "hash", hash, "err", err) return nil } - if len(storageReceipts) != 1 { - log.Error("Invalid bor receipt array RLP", "hash", hash, "err", err) + if nReceipts := len(storageReceipts); nReceipts != 1 { + log.Error("Invalid bor receipt array RLP length", "number", number, "hash", hash, "nReceipts", nReceipts) return nil } return (*types.Receipt)(storageReceipts[0]) From a3ed4eadffcf0e6e92e1dad9d2a280cd22e6821c Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sat, 13 May 2023 17:58:59 +0530 Subject: [PATCH 23/53] internal/cli: add snapshot prune block tests (referenced from bsc's PR) --- internal/cli/snapshot_test.go | 231 ++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 internal/cli/snapshot_test.go diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go new file mode 100644 index 0000000000..e583c20a19 --- /dev/null +++ b/internal/cli/snapshot_test.go @@ -0,0 +1,231 @@ +package cli + +import ( + "bytes" + "encoding/hex" + "fmt" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/pruner" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/cli/server" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + canonicalSeed = 1 + blockPruneBackUpBlockNumber = 128 + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + balance = big.NewInt(1_000000000000000000) + gspec = &core.Genesis{Config: params.TestChainConfig, Alloc: core.GenesisAlloc{address: {Balance: balance}}} + signer = types.LatestSigner(gspec.Config) + config = &core.CacheConfig{ + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 0, // Disable snapshot + TriesInMemory: 128, + } + engine = ethash.NewFullFaker() +) + +func TestOfflineBlockPrune(t *testing.T) { + // Corner case for 0 remain in ancinetStore. + testOfflineBlockPruneWithAmountReserved(t, 0) + // General case. + testOfflineBlockPruneWithAmountReserved(t, 100) +} + +func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64) { + datadir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create temporary datadir: %v", err) + } + os.RemoveAll(datadir) + + chaindbPath := filepath.Join(datadir, "chaindata") + oldAncientPath := filepath.Join(chaindbPath, "ancient") + newAncientPath := filepath.Join(chaindbPath, "ancient_back") + + _, blocks, blockList, receiptsList, externTdList, startBlockNumber, _ := BlockchainCreator(t, chaindbPath, oldAncientPath, amountReserved) + node, _ := startEthService(t, gspec, blocks, chaindbPath) + defer node.Close() + + // Initialize a block pruner for pruning, only remain amountReserved blocks backward. + testBlockPruner := pruner.NewBlockPruner(node, oldAncientPath, newAncientPath, amountReserved) + if err != nil { + t.Fatalf("failed to make new blockpruner: %v", err) + } + dbHandles, err := server.MakeDatabaseHandles() + if err != nil { + t.Fatalf("failed to create db handles: %v", err) + } + if err := testBlockPruner.BlockPruneBackup(chaindbPath, 512, dbHandles, "", false, false); err != nil { + t.Fatalf("Failed to back up block: %v", err) + } + + dbBack, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, newAncientPath, "", false, true, false) + if err != nil { + t.Fatalf("failed to create database with ancient backend") + } + defer dbBack.Close() + + //check against if the backup data matched original one + for blockNumber := startBlockNumber; blockNumber < startBlockNumber+amountReserved; blockNumber++ { + blockHash := rawdb.ReadCanonicalHash(dbBack, blockNumber) + block := rawdb.ReadBlock(dbBack, blockHash, blockNumber) + + if block.Hash() != blockHash { + t.Fatalf("block data did not match between oldDb and backupDb") + } + if blockList[blockNumber-startBlockNumber].Hash() != blockHash { + t.Fatalf("block data did not match between oldDb and backupDb") + } + + receipts := rawdb.ReadRawReceipts(dbBack, blockHash, blockNumber) + if err := checkReceiptsRLP(receipts, receiptsList[blockNumber-startBlockNumber]); err != nil { + t.Fatalf("receipts did not match between oldDb and backupDb") + } + // // Calculate the total difficulty of the block + td := rawdb.ReadTd(dbBack, blockHash, blockNumber) + if td == nil { + t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) + } + if td.Cmp(externTdList[blockNumber-startBlockNumber]) != 0 { + t.Fatalf("externTd did not match between oldDb and backupDb") + } + } + + //check if ancientDb freezer replaced successfully + testBlockPruner.AncientDbReplacer() + if _, err := os.Stat(newAncientPath); err != nil { + if !os.IsNotExist(err) { + t.Fatalf("ancientDb replaced unsuccessfully") + } + } + if _, err := os.Stat(oldAncientPath); err != nil { + t.Fatalf("ancientDb replaced unsuccessfully") + } +} + +func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemain uint64) (ethdb.Database, []*types.Block, []*types.Block, []types.Receipts, []*big.Int, uint64, *core.BlockChain) { + // Create a database with ancient freezer + db, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, AncientPath, "", false, false, false) + if err != nil { + t.Fatalf("failed to create database with ancient backend: %v", err) + } + defer db.Close() + + genesis := gspec.MustCommit(db) + // Initialize a fresh chain with only a genesis block + blockchain, err := core.NewBlockChain(db, config, gspec.Config, engine, vm.Config{}, nil, nil, nil) + if err != nil { + t.Fatalf("Failed to create chain: %v", err) + } + + // Make chain starting from genesis + blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 500, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{0: byte(canonicalSeed), 19: byte(i)}) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1), params.TxGas, big.NewInt(8750000000), nil), signer, key) + if err != nil { + panic(err) + } + block.AddTx(tx) + block.SetDifficulty(big.NewInt(1000000)) + }) + if _, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("Failed to import canonical chain start: %v", err) + } + + // Force run a freeze cycle + type freezer interface { + Freeze(threshold uint64) error + Ancients() (uint64, error) + } + db.(freezer).Freeze(10) + + frozen, err := db.Ancients() + //make sure there're frozen items + if err != nil || frozen == 0 { + t.Fatalf("Failed to import canonical chain start: %v", err) + } + if frozen < blockRemain { + t.Fatalf("block amount is not enough for pruning: %v", err) + } + + oldOffSet := rawdb.ReadOffsetOfCurrentAncientFreezer(db) + // Get the actual start block number. + startBlockNumber := frozen - blockRemain + oldOffSet + // Initialize the slice to buffer the block data left. + blockList := make([]*types.Block, 0, blockPruneBackUpBlockNumber) + receiptsList := make([]types.Receipts, 0, blockPruneBackUpBlockNumber) + externTdList := make([]*big.Int, 0, blockPruneBackUpBlockNumber) + // All ancient data within the most recent 128 blocks write into memory buffer for future new ancient_back directory usage. + for blockNumber := startBlockNumber; blockNumber < frozen+oldOffSet; blockNumber++ { + blockHash := rawdb.ReadCanonicalHash(db, blockNumber) + block := rawdb.ReadBlock(db, blockHash, blockNumber) + blockList = append(blockList, block) + receipts := rawdb.ReadRawReceipts(db, blockHash, blockNumber) + receiptsList = append(receiptsList, receipts) + // Calculate the total difficulty of the block + td := rawdb.ReadTd(db, blockHash, blockNumber) + if td == nil { + t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) + } + externTdList = append(externTdList, td) + } + + return db, blocks, blockList, receiptsList, externTdList, startBlockNumber, blockchain +} + +func checkReceiptsRLP(have, want types.Receipts) error { + if len(have) != len(want) { + return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want)) + } + for i := 0; i < len(want); i++ { + rlpHave, err := rlp.EncodeToBytes(have[i]) + if err != nil { + return err + } + rlpWant, err := rlp.EncodeToBytes(want[i]) + if err != nil { + return err + } + if !bytes.Equal(rlpHave, rlpWant) { + return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) + } + } + return nil +} + +// startEthService creates a full node instance for testing. +func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block, chaindbPath string) (*node.Node, *eth.Ethereum) { + t.Helper() + n, err := node.New(&node.Config{DataDir: chaindbPath}) + if err != nil { + t.Fatal("can't create node:", err) + } + + if err := n.Start(); err != nil { + t.Fatal("can't start node:", err) + } + + return n, nil +} From c314ed86c50427ed03b53242f792f921e5108ee5 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 15 May 2023 16:55:40 +0530 Subject: [PATCH 24/53] improve errors --- core/state/pruner/pruner.go | 22 +++++++++++++++------- internal/cli/snapshot.go | 8 ++++++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index f6028d331a..36dde51a40 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -90,7 +90,7 @@ type Pruner struct { func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) { headBlock := rawdb.ReadHeadBlock(db) if headBlock == nil { - return nil, errors.New("Failed to load head block") + return nil, errors.New("failed to load head block") } snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) if err != nil { @@ -359,7 +359,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // Open old db wrapper. chainDb, err := p.node.OpenDatabaseWithFreezer(name, cache, handles, p.oldAncientPath, namespace, readonly, true, interrupt) if err != nil { - return fmt.Errorf("Failed to open ancient database %v", err) + return fmt.Errorf("failed to open ancient database: %v", err) } defer chainDb.Close() @@ -367,9 +367,14 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str itemsOfAncient, err := chainDb.ItemAmountInAncient() log.Info("ChainDB opened successfully", "itemsOfAncient", itemsOfAncient) - // If we can't access the freezer or it's empty, abort. - if err != nil || itemsOfAncient == 0 { - return errors.New("can't access the freezer or it's empty, abort") + // Abort if we can't access the freezer + if err != nil { + return fmt.Errorf("failed to access the freezer: %v", err) + } + + // Also abort if it's empty + if itemsOfAncient == 0 { + return errors.New("freezer is empty, abort") } // If the items in freezer is less than the block amount that we want to reserve, it is not enough, should stop. @@ -398,7 +403,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // For every round, newoffset actually equals to the startBlockNumber in ancient backup db. frdbBack, err := rawdb.NewFreezerDb(chainDb, p.newAncientPath, namespace, readonly, startBlockNumber) if err != nil { - return fmt.Errorf("Failed to create ancient freezer backup: %v", err) + return fmt.Errorf("failed to create ancient freezer backup: %v", err) } defer frdbBack.Close() @@ -441,7 +446,10 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str start = time.Now() } } - lock.Release() + err = lock.Release() + if err != nil { + return fmt.Errorf("failed to release file lock: %v", err) + } log.Info("Backup old ancientDB done", "current start blockNumber in ancientDB", startBlockNumber) return nil } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 168c8b4609..a5d9b15d3c 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -341,7 +341,7 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) if err != nil { - log.Error("snaptree error", "err", err) + log.Error("Unable to load snapshot", "err", err) return err // The relevant snapshot(s) might not exist } @@ -349,6 +349,8 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // - in most of the normal cases, the related state is available // - the probability of this layer being reorg is very low + // Note that here (n) refers to `c.triesInMemory` which is a + // configurable parameter. // Retrieve all snapshot layers from the current HEAD. // In theory there are n difflayers + 1 disk layer present, // so n diff layers are expected to be returned. @@ -436,7 +438,9 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { return err } if exist { - defer lock.Release() + defer func() { + _ = lock.Release() + }() log.Info("File lock existed, waiting for prune recovery and continue", "err", err) if err := blockpruner.RecoverInterruption("chaindata", c.cache, fdHandles, "", false); err != nil { log.Error("Pruning failed", "err", err) From 98c020c99f1814e4af1da32df734ae2e712397ad Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 15 May 2023 18:54:38 +0530 Subject: [PATCH 25/53] cmd, core, eth, internal: fix lint --- cmd/geth/dbcmd.go | 2 ++ core/blockchain.go | 1 + core/rawdb/bor_receipt.go | 4 ++- core/rawdb/chain_iterator.go | 3 +++ core/rawdb/database.go | 9 +++++++ core/rawdb/freezer_test.go | 4 ++- core/state/pruner/pruner.go | 17 ++++++++++-- eth/downloader/downloader.go | 1 + eth/state_accessor.go | 1 + internal/cli/snapshot.go | 32 ++++++++++++++++++++--- internal/cli/snapshot_test.go | 49 +++++++++++++++++++++++++---------- internal/ethapi/api.go | 1 + 12 files changed, 103 insertions(+), 21 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 30b9ecc78b..cc4bc91e2c 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -639,6 +639,7 @@ func importLDBdata(ctx *cli.Context) error { } close(stop) }() + db := utils.MakeChainDatabase(ctx, stack, false, false) return utils.ImportLDBData(db, fName, int64(start), stop) } @@ -735,6 +736,7 @@ func exportChaindata(ctx *cli.Context) error { } close(stop) }() + db := utils.MakeChainDatabase(ctx, stack, true, false) return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop) } diff --git a/core/blockchain.go b/core/blockchain.go index f5b8545baa..ef11d2a72f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -344,6 +344,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } // Ensure that a previous crash in SetHead doesn't leave extra ancients + //nolint:nestif if frozen, err := bc.db.ItemAmountInAncient(); err == nil && frozen > 0 { frozen, err = bc.db.Ancients() if err != nil { diff --git a/core/rawdb/bor_receipt.go b/core/rawdb/bor_receipt.go index 420fd165b6..8d6ab5d0ef 100644 --- a/core/rawdb/bor_receipt.go +++ b/core/rawdb/bor_receipt.go @@ -64,7 +64,7 @@ func ReadBorReceiptRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.Raw func ReadRawBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types.Receipt { // Retrieve the flattened receipt slice data := ReadBorReceiptRLP(db, hash, number) - if data == nil || len(data) == 0 { + if len(data) == 0 { return nil } @@ -76,10 +76,12 @@ func ReadRawBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types. log.Error("Invalid bor receipt array RLP", "number", number, "hash", hash, "err", err) return nil } + if nReceipts := len(storageReceipts); nReceipts != 1 { log.Error("Invalid bor receipt array RLP length", "number", number, "hash", hash, "nReceipts", nReceipts) return nil } + return (*types.Receipt)(storageReceipts[0]) } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 931e1cb87f..f9840ab7ac 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -45,6 +45,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { hash common.Hash offset = db.AncientOffSet() ) + for i := uint64(0) + offset; i < frozen+offset; { // We read 100K hashes at a time, for a total of 3.2M count := uint64(100_000) @@ -99,9 +100,11 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool number uint64 rlp rlp.RawValue } + if offset := db.AncientOffSet(); offset > from { from = offset } + if to <= from { return nil } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 842c87efbd..cc6e3a1924 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -173,6 +173,7 @@ func ReadOffsetOfCurrentAncientFreezer(db ethdb.KeyValueReader) uint64 { if offset == nil { return 0 } + return new(big.Int).SetBytes(offset).Uint64() } @@ -182,6 +183,7 @@ func ReadOffsetOfLastAncientFreezer(db ethdb.KeyValueReader) uint64 { if offset == nil { return 0 } + return new(big.Int).SetBytes(offset).Uint64() } @@ -206,6 +208,7 @@ func NewFreezerDb(db ethdb.KeyValueStore, frz, namespace string, readonly bool, if err != nil { return nil, err } + return frdb, nil } @@ -249,6 +252,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the genesis hash is empty, we have a new key-value store, so nothing to // validate in this method. If, however, the genesis hash is not nil, compare // it to the freezer content. + //nolint:nestif if kvgenesis, _ := db.Get(headerHashKey(0)); offset == 0 && len(kvgenesis) > 0 { if frozen, _ := frdb.Ancients(); frozen > 0 { // If the freezer already contains something, ensure that the genesis blocks @@ -335,6 +339,7 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer if err != nil { return nil, err } + frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly, disableFreeze, isLastOffset) if err != nil { kvdb.Close() @@ -377,18 +382,22 @@ func AncientInspect(db ethdb.Database) error { offset := counter(ReadOffsetOfCurrentAncientFreezer(db)) // Get number of ancient rows inside the freezer. ancients := counter(0) + if count, err := db.ItemAmountInAncient(); err != nil { log.Error("failed to get the items amount in ancientDB", "err", err) return err } else { ancients = counter(count) } + var endNumber counter + if offset+ancients <= 0 { endNumber = 0 } else { endNumber = offset + ancients - 1 } + stats := [][]string{ {"Offset/StartBlockNumber", "Offset/StartBlockNumber of ancientDB", offset.String()}, {"Amount of remained items in AncientStore", "Remaining items of ancientDB", ancients.String()}, diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index e8857042fb..3268309c17 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -268,11 +268,13 @@ func TestFreezerReadonlyValidate(t *testing.T) { t.Fatal("can't open freezer", err) } var item = make([]byte, 1024) + aBatch := f.tables["a"].newBatch(0) require.NoError(t, aBatch.AppendRaw(0, item)) require.NoError(t, aBatch.AppendRaw(1, item)) require.NoError(t, aBatch.AppendRaw(2, item)) require.NoError(t, aBatch.commit()) + bBatch := f.tables["b"].newBatch(0) require.NoError(t, bBatch.AppendRaw(0, item)) require.NoError(t, bBatch.commit()) @@ -286,7 +288,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { // Re-openening as readonly should fail when validating // table lengths. - f, err = newFreezer(dir, "", true, 0, 2049, tables) + _, err = newFreezer(dir, "", true, 0, 2049, tables) if err == nil { t.Fatal("readonly freezer should fail with differing table lengths") } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 36dde51a40..f86b7dd7ce 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/prometheus/tsdb/fileutil" ) @@ -410,6 +411,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str offsetBatch := chainDb.NewBatch() rawdb.WriteOffsetOfCurrentAncientFreezer(offsetBatch, startBlockNumber) rawdb.WriteOffsetOfLastAncientFreezer(offsetBatch, oldOffset) + if err := offsetBatch.Write(); err != nil { log.Crit("Failed to write offset into disk", "err", err) } @@ -443,14 +445,17 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // Print the log every 5s for better trace. if time.Since(start) > 5*time.Second { log.Info("Block backup process running successfully", "current blockNumber for backup", blockNumber) + start = time.Now() } } - err = lock.Release() - if err != nil { + + if err = lock.Release(); err != nil { return fmt.Errorf("failed to release file lock: %v", err) } + log.Info("Backup old ancientDB done", "current start blockNumber in ancientDB", startBlockNumber) + return nil } @@ -464,18 +469,22 @@ func (p *BlockPruner) BlockPruneBackup(name string, cache, handles int, namespac } log.Info("Block pruning backup successfully", "time duration since start is", common.PrettyDuration(time.Since(start))) + return nil } func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, namespace string, readonly bool) error { log.Info("RecoverInterruption for block prune") + newExist, err := checkFileExist(p.newAncientPath) if err != nil { return fmt.Errorf("newAncientDB path error %v", err) } + //nolint:nestif if newExist { log.Info("New ancientDB_backup existed in interruption scenario") + flockOfAncientBack, err := checkFileExist(filepath.Join(p.newAncientPath, "PRUNEFLOCKBACK")) if err != nil { return fmt.Errorf("failed to check flock of ancientDB_Back %v", err) @@ -486,6 +495,7 @@ func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, names if err := os.RemoveAll(p.newAncientPath); err != nil { return fmt.Errorf("failed to remove old ancient directory %v", err) } + if flockOfAncientBack { // Indicating the oldOffset/newOffset have already been updated. if err := p.BlockPruneBackup(name, cache, handles, namespace, readonly, true); err != nil { @@ -522,8 +532,10 @@ func checkFileExist(path string) (bool, error) { // Indicating the file didn't exist. return false, nil } + return true, err } + return true, nil } @@ -537,6 +549,7 @@ func (p *BlockPruner) AncientDbReplacer() error { if err := os.Rename(p.newAncientPath, p.oldAncientPath); err != nil { return fmt.Errorf("failed to rename new ancient directory %v", err) } + return nil } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 462cd8b818..420bcc188c 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -560,6 +560,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd * } else { d.ancientLimit = 0 } + frozen, _ := d.stateDB.ItemAmountInAncient() // Ignore the error here since light client can also hit here. // If a part of blockchain data has already been written into active store, diff --git a/eth/state_accessor.go b/eth/state_accessor.go index e655510e24..bb4da53b26 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -72,6 +72,7 @@ func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state // The optional base statedb is given, mark the start point as parent block statedb, database, report = base, base.Database(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if current == nil { return nil, fmt.Errorf("missing parent block %v %d", block.ParentHash(), block.NumberU64()-1) } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 2ad5da5208..3e6751f48f 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -18,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/trie" + "github.com/prometheus/tsdb/fileutil" "github.com/mitchellh/cli" @@ -303,7 +304,7 @@ func (c *PruneBlockCommand) Run(args []string) int { } defer node.Close() - dbHandles, err := server.MakeDatabaseHandles() + dbHandles, err := server.MakeDatabaseHandles(0) if err != nil { c.UI.Error(err.Error()) return 1 @@ -320,6 +321,7 @@ func (c *PruneBlockCommand) Run(args []string) int { c.UI.Error(err.Error()) return 1 } + return 0 } @@ -333,10 +335,12 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { if !c.checkSnapshotWithMPT { return nil } + headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { return errors.New("failed to load head block") } + headHeader := headBlock.Header() //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) @@ -368,6 +372,7 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the // entire trie. + //nolint:nestif if blob := rawdb.ReadTrieNode(chaindb, targetRoot); len(blob) == 0 { // The special case is for clique based networks(rinkeby, goerli // and some other private networks), it's possible that two @@ -381,26 +386,33 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // state available, but we don't want to use the topmost state // as the pruning target. var found bool + for i := len(layers) - 2; i >= 1; i-- { if blob := rawdb.ReadTrieNode(chaindb, layers[i].Root()); len(blob) != 0 { targetRoot = layers[i].Root() found = true + log.Info("Selecting middle-layer as the pruning target", "root", targetRoot, "depth", i) + break } } + if !found { if blob := rawdb.ReadTrieNode(chaindb, snaptree.DiskRoot()); len(blob) != 0 { targetRoot = snaptree.DiskRoot() found = true + log.Info("Selecting disk-layer as the pruning target", "root", targetRoot) } } + if !found { if len(layers) > 0 { log.Error("no snapshot paired state") return errors.New("no snapshot paired state") } + return fmt.Errorf("associated state[%x] is not present", targetRoot) } } else { @@ -410,6 +422,7 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { log.Info("Selecting user-specified state as the pruning target", "root", targetRoot) } } + return nil } @@ -417,6 +430,7 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { name := "chaindata" oldAncientPath := c.datadirAncient + switch { case oldAncientPath == "": oldAncientPath = filepath.Join(stack.ResolvePath(name), "ancient") @@ -428,8 +442,8 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { if path == "" { return errors.New("prune failed, did not specify the AncientPath") } - newAncientPath := filepath.Join(path, "ancient_back") + newAncientPath := filepath.Join(path, "ancient_back") blockpruner := pruner.NewBlockPruner(stack, oldAncientPath, newAncientPath, c.blockAmountReserved) lock, exist, err := fileutil.Flock(filepath.Join(oldAncientPath, "PRUNEFLOCK")) @@ -437,16 +451,20 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { log.Error("file lock error", "err", err) return err } + if exist { defer func() { _ = lock.Release() }() log.Info("File lock existed, waiting for prune recovery and continue", "err", err) + if err := blockpruner.RecoverInterruption("chaindata", c.cache, fdHandles, "", false); err != nil { log.Error("Pruning failed", "err", err) return err } + log.Info("Block prune successfully") + return nil } @@ -456,9 +474,12 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { if err := blockpruner.AncientDbReplacer(); err != nil { return err } + log.Info("Block prune successfully") + return nil } + if err := blockpruner.BlockPruneBackup(name, c.cache, fdHandles, "", false, false); err != nil { return err } @@ -470,7 +491,12 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { return err } - lock.Release() + if err = lock.Release(); err != nil { + log.Error("Unable to release lock on file", "err", err) + + return err + } + log.Info("Block prune successfully") return nil diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index e583c20a19..602cdfaec5 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" "math/big" "os" "path/filepath" @@ -20,7 +19,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/cli/server" "github.com/ethereum/go-ethereum/node" @@ -47,6 +45,8 @@ var ( ) func TestOfflineBlockPrune(t *testing.T) { + t.Parallel() + // Corner case for 0 remain in ancinetStore. testOfflineBlockPruneWithAmountReserved(t, 0) // General case. @@ -54,29 +54,32 @@ func TestOfflineBlockPrune(t *testing.T) { } func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64) { - datadir, err := ioutil.TempDir("", "") + t.Helper() + + datadir, err := os.MkdirTemp("", "") if err != nil { t.Fatalf("Failed to create temporary datadir: %v", err) } + os.RemoveAll(datadir) chaindbPath := filepath.Join(datadir, "chaindata") oldAncientPath := filepath.Join(chaindbPath, "ancient") newAncientPath := filepath.Join(chaindbPath, "ancient_back") - _, blocks, blockList, receiptsList, externTdList, startBlockNumber, _ := BlockchainCreator(t, chaindbPath, oldAncientPath, amountReserved) - node, _ := startEthService(t, gspec, blocks, chaindbPath) + _, _, blockList, receiptsList, externTdList, startBlockNumber, _ := BlockchainCreator(t, chaindbPath, oldAncientPath, amountReserved) + + node := startEthService(t, chaindbPath) defer node.Close() // Initialize a block pruner for pruning, only remain amountReserved blocks backward. testBlockPruner := pruner.NewBlockPruner(node, oldAncientPath, newAncientPath, amountReserved) - if err != nil { - t.Fatalf("failed to make new blockpruner: %v", err) - } - dbHandles, err := server.MakeDatabaseHandles() + dbHandles, err := server.MakeDatabaseHandles(0) + if err != nil { t.Fatalf("failed to create db handles: %v", err) } + if err := testBlockPruner.BlockPruneBackup(chaindbPath, 512, dbHandles, "", false, false); err != nil { t.Fatalf("Failed to back up block: %v", err) } @@ -95,6 +98,7 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 if block.Hash() != blockHash { t.Fatalf("block data did not match between oldDb and backupDb") } + if blockList[blockNumber-startBlockNumber].Hash() != blockHash { t.Fatalf("block data did not match between oldDb and backupDb") } @@ -108,24 +112,31 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 if td == nil { t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) } + if td.Cmp(externTdList[blockNumber-startBlockNumber]) != 0 { t.Fatalf("externTd did not match between oldDb and backupDb") } } - //check if ancientDb freezer replaced successfully - testBlockPruner.AncientDbReplacer() + // Check if ancientDb freezer replaced successfully + if err = testBlockPruner.AncientDbReplacer(); err != nil { + t.Fatalf("error in replacing ancient db") + } + if _, err := os.Stat(newAncientPath); err != nil { if !os.IsNotExist(err) { t.Fatalf("ancientDb replaced unsuccessfully") } } + if _, err := os.Stat(oldAncientPath); err != nil { t.Fatalf("ancientDb replaced unsuccessfully") } } func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemain uint64) (ethdb.Database, []*types.Block, []*types.Block, []types.Receipts, []*big.Int, uint64, *core.BlockChain) { + t.Helper() + // Create a database with ancient freezer db, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, AncientPath, "", false, false, false) if err != nil { @@ -159,13 +170,17 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai Freeze(threshold uint64) error Ancients() (uint64, error) } - db.(freezer).Freeze(10) + + if err = db.(freezer).Freeze(10); err != nil { + t.Fatalf("Error in freeze operation: %v", err) + } frozen, err := db.Ancients() //make sure there're frozen items if err != nil || frozen == 0 { t.Fatalf("Failed to import canonical chain start: %v", err) } + if frozen < blockRemain { t.Fatalf("block amount is not enough for pruning: %v", err) } @@ -189,6 +204,7 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai if td == nil { t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) } + externTdList = append(externTdList, td) } @@ -199,25 +215,30 @@ func checkReceiptsRLP(have, want types.Receipts) error { if len(have) != len(want) { return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want)) } + for i := 0; i < len(want); i++ { rlpHave, err := rlp.EncodeToBytes(have[i]) if err != nil { return err } + rlpWant, err := rlp.EncodeToBytes(want[i]) if err != nil { return err } + if !bytes.Equal(rlpHave, rlpWant) { return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) } } + return nil } // startEthService creates a full node instance for testing. -func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block, chaindbPath string) (*node.Node, *eth.Ethereum) { +func startEthService(t *testing.T, chaindbPath string) *node.Node { t.Helper() + n, err := node.New(&node.Config{DataDir: chaindbPath}) if err != nil { t.Fatal("can't create node:", err) @@ -227,5 +248,5 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block, t.Fatal("can't start node:", err) } - return n, nil + return n } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f1ba70a491..cb56e41a86 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -636,6 +636,7 @@ func (s *PublicBlockChainAPI) GetTransactionReceiptsByBlock(ctx context.Context, if err != nil { return nil, err } + if receipts == nil { return nil, fmt.Errorf("block %d receipts not found", block.NumberU64()) } From 3d71ce570b0385837c8215e77109161cd5dbe072 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 16 May 2023 09:45:43 +0530 Subject: [PATCH 26/53] internal/cli: refactor snapshot prune block test --- internal/cli/snapshot_test.go | 124 +++++++++++----------------------- 1 file changed, 41 insertions(+), 83 deletions(-) diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index 602cdfaec5..a8765d095c 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -2,8 +2,6 @@ package cli import ( "bytes" - "encoding/hex" - "fmt" "math/big" "os" "path/filepath" @@ -24,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/stretchr/testify/require" ) var ( @@ -49,6 +48,7 @@ func TestOfflineBlockPrune(t *testing.T) { // Corner case for 0 remain in ancinetStore. testOfflineBlockPruneWithAmountReserved(t, 0) + // General case. testOfflineBlockPruneWithAmountReserved(t, 100) } @@ -57,9 +57,7 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 t.Helper() datadir, err := os.MkdirTemp("", "") - if err != nil { - t.Fatalf("Failed to create temporary datadir: %v", err) - } + require.NoError(t, err, "failed to create temporary datadir") os.RemoveAll(datadir) @@ -75,19 +73,13 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 // Initialize a block pruner for pruning, only remain amountReserved blocks backward. testBlockPruner := pruner.NewBlockPruner(node, oldAncientPath, newAncientPath, amountReserved) dbHandles, err := server.MakeDatabaseHandles(0) + require.NoError(t, err, "failed to create database handles") - if err != nil { - t.Fatalf("failed to create db handles: %v", err) - } - - if err := testBlockPruner.BlockPruneBackup(chaindbPath, 512, dbHandles, "", false, false); err != nil { - t.Fatalf("Failed to back up block: %v", err) - } + err = testBlockPruner.BlockPruneBackup(chaindbPath, 512, dbHandles, "", false, false) + require.NoError(t, err, "failed to backup block") dbBack, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, newAncientPath, "", false, true, false) - if err != nil { - t.Fatalf("failed to create database with ancient backend") - } + require.NoError(t, err, "failed to create db with ancient backend") defer dbBack.Close() //check against if the backup data matched original one @@ -95,33 +87,22 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 blockHash := rawdb.ReadCanonicalHash(dbBack, blockNumber) block := rawdb.ReadBlock(dbBack, blockHash, blockNumber) - if block.Hash() != blockHash { - t.Fatalf("block data did not match between oldDb and backupDb") - } - - if blockList[blockNumber-startBlockNumber].Hash() != blockHash { - t.Fatalf("block data did not match between oldDb and backupDb") - } + require.Equal(t, block.Hash(), blockHash, "block data mismatch between oldDb and backupDb") + require.Equal(t, blockList[blockNumber-startBlockNumber].Hash(), blockHash, "block data mismatch between oldDb and backupDb") receipts := rawdb.ReadRawReceipts(dbBack, blockHash, blockNumber) - if err := checkReceiptsRLP(receipts, receiptsList[blockNumber-startBlockNumber]); err != nil { - t.Fatalf("receipts did not match between oldDb and backupDb") - } - // // Calculate the total difficulty of the block + checkReceiptsRLP(t, receipts, receiptsList[blockNumber-startBlockNumber]) + + // Calculate the total difficulty of the block td := rawdb.ReadTd(dbBack, blockHash, blockNumber) - if td == nil { - t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) - } + require.NotNil(t, td, "failed to read td", consensus.ErrUnknownAncestor) - if td.Cmp(externTdList[blockNumber-startBlockNumber]) != 0 { - t.Fatalf("externTd did not match between oldDb and backupDb") - } + require.Equal(t, td.Cmp(externTdList[blockNumber-startBlockNumber]), 0, "Td mismatch between oldDb and backupDb") } // Check if ancientDb freezer replaced successfully - if err = testBlockPruner.AncientDbReplacer(); err != nil { - t.Fatalf("error in replacing ancient db") - } + err = testBlockPruner.AncientDbReplacer() + require.NoError(t, err, "error replacing ancient db") if _, err := os.Stat(newAncientPath); err != nil { if !os.IsNotExist(err) { @@ -129,9 +110,8 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 } } - if _, err := os.Stat(oldAncientPath); err != nil { - t.Fatalf("ancientDb replaced unsuccessfully") - } + _, err = os.Stat(oldAncientPath) + require.NoError(t, err, "failed to replace ancientDb") } func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemain uint64) (ethdb.Database, []*types.Block, []*types.Block, []types.Receipts, []*big.Int, uint64, *core.BlockChain) { @@ -139,31 +119,27 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai // Create a database with ancient freezer db, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, AncientPath, "", false, false, false) - if err != nil { - t.Fatalf("failed to create database with ancient backend: %v", err) - } + require.NoError(t, err, "failed to create db with ancient backend") defer db.Close() genesis := gspec.MustCommit(db) // Initialize a fresh chain with only a genesis block blockchain, err := core.NewBlockChain(db, config, gspec.Config, engine, vm.Config{}, nil, nil, nil) - if err != nil { - t.Fatalf("Failed to create chain: %v", err) - } + require.NoError(t, err, "failed to create chain") // Make chain starting from genesis blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 500, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{0: byte(canonicalSeed), 19: byte(i)}) tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1), params.TxGas, big.NewInt(8750000000), nil), signer, key) if err != nil { - panic(err) + require.NoError(t, err, "failed to sign tx") } block.AddTx(tx) block.SetDifficulty(big.NewInt(1000000)) }) - if _, err := blockchain.InsertChain(blocks); err != nil { - t.Fatalf("Failed to import canonical chain start: %v", err) - } + + _, err = blockchain.InsertChain(blocks) + require.NoError(t, err, "failed to insert chain") // Force run a freeze cycle type freezer interface { @@ -171,19 +147,14 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai Ancients() (uint64, error) } - if err = db.(freezer).Freeze(10); err != nil { - t.Fatalf("Error in freeze operation: %v", err) - } + err = db.(freezer).Freeze(10) + require.NoError(t, err, "failed to perform freeze operation") + // make sure there're frozen items frozen, err := db.Ancients() - //make sure there're frozen items - if err != nil || frozen == 0 { - t.Fatalf("Failed to import canonical chain start: %v", err) - } - - if frozen < blockRemain { - t.Fatalf("block amount is not enough for pruning: %v", err) - } + require.NoError(t, err, "failed to fetch ancients items from db") + require.NotEqual(t, frozen, 0, "no elements in freezer db") + require.GreaterOrEqual(t, frozen, blockRemain, "block amount is not enough for pruning") oldOffSet := rawdb.ReadOffsetOfCurrentAncientFreezer(db) // Get the actual start block number. @@ -201,9 +172,7 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai receiptsList = append(receiptsList, receipts) // Calculate the total difficulty of the block td := rawdb.ReadTd(db, blockHash, blockNumber) - if td == nil { - t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) - } + require.NotNil(t, td, "failed to read td", consensus.ErrUnknownAncestor) externTdList = append(externTdList, td) } @@ -211,28 +180,20 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai return db, blocks, blockList, receiptsList, externTdList, startBlockNumber, blockchain } -func checkReceiptsRLP(have, want types.Receipts) error { - if len(have) != len(want) { - return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want)) - } +func checkReceiptsRLP(t *testing.T, have, want types.Receipts) { + t.Helper() + + require.Equal(t, len(want), len(have), "receipts sizes mismatch") for i := 0; i < len(want); i++ { rlpHave, err := rlp.EncodeToBytes(have[i]) - if err != nil { - return err - } + require.NoError(t, err, "error in rlp encoding") rlpWant, err := rlp.EncodeToBytes(want[i]) - if err != nil { - return err - } + require.NoError(t, err, "error in rlp encoding") - if !bytes.Equal(rlpHave, rlpWant) { - return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) - } + require.Equal(t, true, bytes.Equal(rlpHave, rlpWant), "receipt rlp mismatch") } - - return nil } // startEthService creates a full node instance for testing. @@ -240,13 +201,10 @@ func startEthService(t *testing.T, chaindbPath string) *node.Node { t.Helper() n, err := node.New(&node.Config{DataDir: chaindbPath}) - if err != nil { - t.Fatal("can't create node:", err) - } + require.NoError(t, err, "failed to create node") - if err := n.Start(); err != nil { - t.Fatal("can't start node:", err) - } + err = n.Start() + require.NoError(t, err, "failed to start node") return n } From f195feb7b23061a68f11d31184d8ef2b038ab4c2 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 16 May 2023 11:02:59 +0530 Subject: [PATCH 27/53] fix linters in tests --- internal/cli/snapshot_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index a8765d095c..d0eff67346 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/stretchr/testify/require" ) @@ -80,6 +81,7 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 dbBack, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, newAncientPath, "", false, true, false) require.NoError(t, err, "failed to create db with ancient backend") + defer dbBack.Close() //check against if the backup data matched original one @@ -120,6 +122,7 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai // Create a database with ancient freezer db, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, AncientPath, "", false, false, false) require.NoError(t, err, "failed to create db with ancient backend") + defer db.Close() genesis := gspec.MustCommit(db) From 7104f955f54bcba21b3e5f045ba39cb502674029 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 18 May 2023 01:22:06 +0530 Subject: [PATCH 28/53] internal/cli: add inspect-ancient-db command, update docs --- docs/cli/README.md | 2 + docs/cli/bootnode.md | 2 +- docs/cli/debug_pprof.md | 2 +- docs/cli/inspect-ancient-db.md | 19 ++++++ docs/cli/snapshot.md | 4 +- internal/cli/command.go | 5 ++ internal/cli/snapshot.go | 108 ++++++++++++++++++++++++++++++++- 7 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 docs/cli/inspect-ancient-db.md diff --git a/docs/cli/README.md b/docs/cli/README.md index d97e786686..22dbe43bed 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -30,6 +30,8 @@ - [```fingerprint```](./fingerprint.md) +- [```inspect-ancient-db```](./inspect-ancient-db.md) + - [```peers```](./peers.md) - [```peers add```](./peers_add.md) diff --git a/docs/cli/bootnode.md b/docs/cli/bootnode.md index 465eadd9ff..e4111160a0 100644 --- a/docs/cli/bootnode.md +++ b/docs/cli/bootnode.md @@ -16,4 +16,4 @@ - ```save-key```: path to save the ecdsa private key -- ```dry-run```: validates parameters and prints bootnode configurations, but does not start bootnode (default: false) +- ```dry-run```: validates parameters and prints bootnode configurations, but does not start bootnode (default: false) \ No newline at end of file diff --git a/docs/cli/debug_pprof.md b/docs/cli/debug_pprof.md index fbc9a70c52..78e75f2134 100644 --- a/docs/cli/debug_pprof.md +++ b/docs/cli/debug_pprof.md @@ -10,4 +10,4 @@ The ```debug pprof ``` command will create an archive containing bor ppro - ```output```: Output directory -- ```skiptrace```: Skip running the trace (default: false) +- ```skiptrace```: Skip running the trace (default: false) \ No newline at end of file diff --git a/docs/cli/inspect-ancient-db.md b/docs/cli/inspect-ancient-db.md new file mode 100644 index 0000000000..0029aaa746 --- /dev/null +++ b/docs/cli/inspect-ancient-db.md @@ -0,0 +1,19 @@ +# Inspect ancient DB for block pruning + +The ```bor snapshot inspect-ancient-db``` command will inspect few fields in the ancient datastore using the given datadir location. + + +This command prints the following information which is useful for block-pruning rounds: + +1. Offset / Start block number (from kvDB). +2. Amount of items in the ancientdb. +3. Last block number written in ancientdb. + + +## Options + +- ```datadir```: Path of the data directory to store information + +- ```keystore```: Path of the data directory to store keys + +- ```datadir.ancient```: Path of the old ancient data directory \ No newline at end of file diff --git a/docs/cli/snapshot.md b/docs/cli/snapshot.md index cc7a908088..ac1bc1ce74 100644 --- a/docs/cli/snapshot.md +++ b/docs/cli/snapshot.md @@ -4,4 +4,6 @@ The ```snapshot``` command groups snapshot related actions: - [```snapshot prune-state```](./snapshot_prune-state.md): Prune state databases at the given datadir location. -- [```snapshot prune-block```](./snapshot_prune-block.md): Prune ancient chaindata at the given datadir location. \ No newline at end of file +- [```snapshot prune-block```](./snapshot_prune-block.md): Prune ancient chaindata at the given datadir location. + +- [```snapshot inspect-ancient-db```](./snapshot_inspect-ancient-db.md): Inspect few fields in ancient datastore. \ No newline at end of file diff --git a/internal/cli/command.go b/internal/cli/command.go index ff620798ff..02a2f35fe2 100644 --- a/internal/cli/command.go +++ b/internal/cli/command.go @@ -204,6 +204,11 @@ func Commands() map[string]MarkDownCommandFactory { Meta: meta, }, nil }, + "inspect-ancient-db": func() (MarkDownCommand, error) { + return &InspectAncientDbCommand{ + Meta: meta, + }, nil + }, } } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 3e6751f48f..cf94f7a789 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -36,6 +36,7 @@ func (a *SnapshotCommand) MarkDown() string { "The ```snapshot``` command groups snapshot related actions:", "- [```snapshot prune-state```](./snapshot_prune-state.md): Prune state databases at the given datadir location.", "- [```snapshot prune-block```](./snapshot_prune-block.md): Prune ancient chaindata at the given datadir location.", + "- [```snapshot inspect-ancient-db```](./snapshot_inspect-ancient-db.md): Inspect few fields in ancient datastore.", } return strings.Join(items, "\n\n") @@ -53,7 +54,11 @@ func (c *SnapshotCommand) Help() string { Prune the ancient data: - $ bor snapshot prune-block` + $ bor snapshot prune-block + + Inspect ancient DB pruning related fields: + + $ bor snapshot inspect-ancient-db` } // Synopsis implements the cli.Command interface @@ -501,3 +506,104 @@ func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { return nil } + +type InspectAncientDbCommand struct { + *Meta + + datadirAncient string +} + +// MarkDown implements cli.MarkDown interface +func (c *InspectAncientDbCommand) MarkDown() string { + items := []string{ + "# Inspect ancient DB for block pruning", + "The ```bor snapshot inspect-ancient-db``` command will inspect few fields in the ancient datastore using the given datadir location.", + ` +This command prints the following information which is useful for block-pruning rounds: + +1. Offset / Start block number (from kvDB). +2. Amount of items in the ancientdb. +3. Last block number written in ancientdb. +`, + c.Flags().MarkDown(), + } + + return strings.Join(items, "\n\n") +} + +// Help implements the cli.Command interface +func (c *InspectAncientDbCommand) Help() string { + return `Usage: bor snapshot inspect-ancient-db + + This command will inspect few fields in the ancient datastore using the given datadir location` + c.Flags().Help() +} + +// Synopsis implements the cli.Command interface +func (c *InspectAncientDbCommand) Synopsis() string { + return "Inspect fields in the ancient blockchain data" +} + +// Flags: datadir, datadir.ancient, cache.trie.journal, bloomfilter.size +func (c *InspectAncientDbCommand) Flags() *flagset.Flagset { + flags := c.NewFlagSet("inspect-ancient-db") + + flags.StringFlag(&flagset.StringFlag{ + Name: "datadir.ancient", + Value: &c.datadirAncient, + Usage: "Path of the old ancient data directory", + Default: "", + }) + + return flags +} + +// Run implements the cli.Command interface +func (c *InspectAncientDbCommand) Run(args []string) int { + flags := c.Flags() + + if err := flags.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + datadir := c.dataDir + if datadir == "" { + c.UI.Error("datadir is required") + return 1 + } + + // Create the node + node, err := node.New(&node.Config{ + DataDir: datadir, + }) + + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + defer node.Close() + + dbHandles, err := server.MakeDatabaseHandles(0) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + err = c.inspectAncientDb(node, dbHandles) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + return 0 +} + +func (c *InspectAncientDbCommand) inspectAncientDb(stack *node.Node, dbHandles int) error { + chaindb, err := stack.OpenDatabaseWithFreezer(chaindataPath, 1024, dbHandles, c.datadirAncient, "", false, true, false) + if err != nil { + return err + } + defer chaindb.Close() + + return rawdb.AncientInspect(chaindb) +} From 54d5a5e34e7fc3f762252c3900b7ac92c381a148 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Fri, 19 May 2023 13:10:00 +0530 Subject: [PATCH 29/53] pruner: use a generic function for simplification --- core/state/pruner/pruner.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index f86b7dd7ce..cdeee3462d 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -425,9 +425,8 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str log.Info("Prune info", "old offset", oldOffset, "number of items in ancientDB", itemsOfAncient, "number of blocks to reserve", p.BlockAmountReserved) log.Info("Record newOffset/newStartBlockNumber successfully", "newOffset", startBlockNumber) - start := time.Now() - // All ancient data after and including startBlockNumber should write into new ancientDB ancient_back. - for blockNumber := startBlockNumber; blockNumber < itemsOfAncient+oldOffset; blockNumber++ { + writeBlock := func(blockNumber uint64) error { + // Read all block data blockHash := rawdb.ReadCanonicalHash(chainDb, blockNumber) block := rawdb.ReadBlock(chainDb, blockHash, blockNumber) receipts := rawdb.ReadRawReceipts(chainDb, blockHash, blockNumber) @@ -438,10 +437,23 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str if td == nil { return consensus.ErrUnknownAncestor } + // Write into new ancient_back db. if _, err := rawdb.WriteAncientBlocks(frdbBack, []*types.Block{block}, []types.Receipts{receipts}, []types.Receipts{borReceipts}, td); err != nil { return fmt.Errorf("failed to write new ancient error: %v", err) } + + return nil + } + + start := time.Now() + // All ancient data after and including startBlockNumber should write into new ancientDB ancient_back. + for blockNumber := startBlockNumber; blockNumber < itemsOfAncient+oldOffset; blockNumber++ { + err := writeBlock(blockNumber) + if err != nil { + return err + } + // Print the log every 5s for better trace. if time.Since(start) > 5*time.Second { log.Info("Block backup process running successfully", "current blockNumber for backup", blockNumber) @@ -550,6 +562,8 @@ func (p *BlockPruner) AncientDbReplacer() error { return fmt.Errorf("failed to rename new ancient directory %v", err) } + log.Info("Replaced existing ancient db with pruned one") + return nil } From 98149be52bc08f3866a25c79439cd1b406107994 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Fri, 19 May 2023 13:10:25 +0530 Subject: [PATCH 30/53] internal/cli: fixes for inspect-db command --- internal/cli/command.go | 2 +- internal/cli/snapshot.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cli/command.go b/internal/cli/command.go index 02a2f35fe2..c72d4bd42d 100644 --- a/internal/cli/command.go +++ b/internal/cli/command.go @@ -204,7 +204,7 @@ func Commands() map[string]MarkDownCommandFactory { Meta: meta, }, nil }, - "inspect-ancient-db": func() (MarkDownCommand, error) { + "snapshot inspect-ancient-db": func() (MarkDownCommand, error) { return &InspectAncientDbCommand{ Meta: meta, }, nil diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index cf94f7a789..f517d07ff9 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -56,9 +56,9 @@ func (c *SnapshotCommand) Help() string { $ bor snapshot prune-block - Inspect ancient DB pruning related fields: + Inspect ancient DB pruning related fields: - $ bor snapshot inspect-ancient-db` + $ bor snapshot inspect-ancient-db` } // Synopsis implements the cli.Command interface From 7c0babac3c130993565093f69b1e5d05674b7199 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Fri, 19 May 2023 13:10:49 +0530 Subject: [PATCH 31/53] internal/cli: improve pruning tests --- internal/cli/snapshot_test.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index d0eff67346..880dcc0da1 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -84,19 +84,32 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 defer dbBack.Close() - //check against if the backup data matched original one + // Check the absence of genesis + genesis, err := dbBack.Ancient("hashes", 0) + require.Equal(t, []byte(nil), genesis, "got genesis but should be absent") + require.NotNil(t, err, "not-nill error expected") + + // Check against if the backup data matched original one for blockNumber := startBlockNumber; blockNumber < startBlockNumber+amountReserved; blockNumber++ { - blockHash := rawdb.ReadCanonicalHash(dbBack, blockNumber) - block := rawdb.ReadBlock(dbBack, blockHash, blockNumber) + // Fetch the data explicitly from ancient db instead of `ReadCanonicalHash` because it + // will pull data from leveldb if not found in ancient. + blockHash, err := dbBack.Ancient("hashes", blockNumber) + require.NoError(t, err, "error fetching block hash from ancient db") + + // We can proceed with fetching other things via generic functions because if + // the block wouldn't have been there in ancient db, the function above to get + // block hash itself would've thrown error. + hash := common.BytesToHash(blockHash) + block := rawdb.ReadBlock(dbBack, hash, blockNumber) - require.Equal(t, block.Hash(), blockHash, "block data mismatch between oldDb and backupDb") - require.Equal(t, blockList[blockNumber-startBlockNumber].Hash(), blockHash, "block data mismatch between oldDb and backupDb") + require.Equal(t, block.Hash(), hash, "block data mismatch between oldDb and backupDb") + require.Equal(t, blockList[blockNumber-startBlockNumber].Hash(), hash, "block data mismatch between oldDb and backupDb") - receipts := rawdb.ReadRawReceipts(dbBack, blockHash, blockNumber) + receipts := rawdb.ReadRawReceipts(dbBack, hash, blockNumber) checkReceiptsRLP(t, receipts, receiptsList[blockNumber-startBlockNumber]) // Calculate the total difficulty of the block - td := rawdb.ReadTd(dbBack, blockHash, blockNumber) + td := rawdb.ReadTd(dbBack, hash, blockNumber) require.NotNil(t, td, "failed to read td", consensus.ErrUnknownAncestor) require.Equal(t, td.Cmp(externTdList[blockNumber-startBlockNumber]), 0, "Td mismatch between oldDb and backupDb") From 6276de6cb208856743934b582825203e281e3bba Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 21 May 2023 12:08:33 +0530 Subject: [PATCH 32/53] core/rawdb: update end block calculation logic in inspect command --- core/rawdb/database.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index cc6e3a1924..95fb339b06 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -395,17 +395,21 @@ func AncientInspect(db ethdb.Database) error { if offset+ancients <= 0 { endNumber = 0 } else { - endNumber = offset + ancients - 1 + sum := counter(0) + if ancients != 0 { + sum = counter(ancients) - 1 + } + endNumber = offset + sum } stats := [][]string{ - {"Offset/StartBlockNumber", "Offset/StartBlockNumber of ancientDB", offset.String()}, - {"Amount of remained items in AncientStore", "Remaining items of ancientDB", ancients.String()}, - {"The last BlockNumber within ancientDB", "The last BlockNumber", endNumber.String()}, + {"Start block number of ancientDB (offset)", offset.String()}, + {"End block number of ancientDB", endNumber.String()}, + {"Remaining items in ancientDB", ancients.String()}, } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Database", "Category", "Items"}) - table.SetFooter([]string{"", "AncientStore information", ""}) + table.SetHeader([]string{"Field", "Items"}) + table.SetFooter([]string{"AncientStore information", ""}) table.AppendBulk(stats) table.Render() From e21c500ca6878545af9f591b4741fee87e1f757a Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 21 May 2023 12:49:14 +0530 Subject: [PATCH 33/53] core/rawdb: improve checks db initialisation --- core/rawdb/database.go | 69 ++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 95fb339b06..16d5212524 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -215,6 +215,8 @@ func NewFreezerDb(db ethdb.KeyValueStore, frz, namespace string, readonly bool, // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold // storage. +// +//nolint:gocognit func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly, disableFreeze, isLastOffset bool) (ethdb.Database, error) { var offset uint64 // The offset of ancientDB should be handled differently in different scenarios. @@ -258,19 +260,30 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. - frgenesis, err := frdb.Ancient(freezerHashTable, 0) - if err != nil { - return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) - } else if !bytes.Equal(kvgenesis, frgenesis) { - return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) + // Only validate genesis if we have `offset` set to 0, which means ancient db pruning + // hasn't happened yet. If the pruning would've happened, genesis would have been wiped + // from ancient db. + if offset == 0 { + frgenesis, err := frdb.Ancient(freezerHashTable, 0) + if err != nil { + return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) + } else if !bytes.Equal(kvgenesis, frgenesis) { + return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis) + } } + // Key-value store and freezer belong to the same network. Ensure that they // are contiguous, otherwise we might end up with a non-functional freezer. - if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { + // + // If ancient db pruning has happened, the number of items in ancient db should + // be less and hence we need to calculate the first block of leveldb by adding + // the offset to it i.e. start block of leveldb = frozen + offset. + startBlock := frozen + offset + if kvhash, _ := db.Get(headerHashKey(startBlock)); len(kvhash) == 0 { // Subsequent header after the freezer limit is missing from the database. - // Reject startup is the database has a more recent head. - if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { - return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) + // Reject startup if the database has a more recent head. + if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > startBlock-1 { + return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", startBlock) } // Database contains only older data than the freezer, this happens if the // state was wiped and reinited from an existing freezer. @@ -279,20 +292,32 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // We might have duplicate blocks (crash after freezer write but before key-value // store deletion, but that's fine). } else { - // If the freezer is empty, ensure nothing was moved yet from the key-value - // store, otherwise we'll end up missing data. We check block #1 to decide - // if we froze anything previously or not, but do take care of databases with - // only the genesis block. - if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { - // Key-value store contains more data than the genesis block, make sure we - // didn't freeze anything yet. - if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { - return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") + // This case means the freezer is empty. Either nothing is moved from the + // key-value store or we've pruned all data. + + // No pruning case + if offset == 0 { + // If the freezer is empty, ensure nothing was moved yet from the key-value + // store, otherwise we'll end up missing data. We check block #1 to decide + // if we froze anything previously or not, but do take care of databases with + // only the genesis block. + if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) { + // Key-value store contains more data than the genesis block, make sure we + // didn't freeze anything yet. + if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { + return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") + } + // Block #1 is still in the database, we're allowed to init a new feezer + } + // Otherwise, the head header is still the genesis, we're allowed to init a new + // freezer. + } else { + // Full pruning case. Check if the key-value store isn't missing any block. + if kvhash, _ := db.Get(headerHashKey(offset)); len(kvhash) == 0 { + log.Error("Missing blocks from leveldb post ancientdb pruning", "ancient chain end block", offset) + return nil, fmt.Errorf("missing blocks from leveldb") } - // Block #1 is still in the database, we're allowed to init a new feezer } - // Otherwise, the head header is still the genesis, we're allowed to init a new - // freezer. } } // Freezer is consistent with the key-value database, permit combining the two @@ -397,7 +422,7 @@ func AncientInspect(db ethdb.Database) error { } else { sum := counter(0) if ancients != 0 { - sum = counter(ancients) - 1 + sum = ancients - 1 } endNumber = offset + sum } From 35a95b2974aed0d0cf62580fe720c99f5d458740 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 21 May 2023 12:49:51 +0530 Subject: [PATCH 34/53] core/rawdb: remove offset check --- core/rawdb/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 16d5212524..8e45586dab 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -255,7 +255,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // validate in this method. If, however, the genesis hash is not nil, compare // it to the freezer content. //nolint:nestif - if kvgenesis, _ := db.Get(headerHashKey(0)); offset == 0 && len(kvgenesis) > 0 { + if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { if frozen, _ := frdb.Ancients(); frozen > 0 { // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both From 495e5cccf76cdbd987a390c010f4d79b95bf1106 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 22 May 2023 19:18:23 +0530 Subject: [PATCH 35/53] update mocks for span, ethdb and add command in makefile --- Makefile | 1 + consensus/bor/span_mock.go | 34 +++++++++++++++++----------------- eth/filters/IDatabase.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 0527284aa8..2c5724220a 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ protoc: generate-mocks: go generate mockgen -destination=./tests/bor/mocks/IHeimdallClient.go -package=mocks ./consensus/bor IHeimdallClient + go generate mockgen -destination=./eth/filters/IDatabase.go -package=filters ./ethdb Database go generate mockgen -destination=./eth/filters/IBackend.go -package=filters ./eth/filters Backend geth: diff --git a/consensus/bor/span_mock.go b/consensus/bor/span_mock.go index 910e81716c..099807161c 100644 --- a/consensus/bor/span_mock.go +++ b/consensus/bor/span_mock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: consensus/bor/span.go +// Source: github.com/ethereum/go-ethereum/consensus/bor (interfaces: Spanner) // Package bor is a generated GoMock package. package bor @@ -42,60 +42,60 @@ func (m *MockSpanner) EXPECT() *MockSpannerMockRecorder { } // CommitSpan mocks base method. -func (m *MockSpanner) CommitSpan(ctx context.Context, heimdallSpan span.HeimdallSpan, state *state.StateDB, header *types.Header, chainContext core.ChainContext) error { +func (m *MockSpanner) CommitSpan(arg0 context.Context, arg1 span.HeimdallSpan, arg2 *state.StateDB, arg3 *types.Header, arg4 core.ChainContext) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CommitSpan", ctx, heimdallSpan, state, header, chainContext) + ret := m.ctrl.Call(m, "CommitSpan", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) return ret0 } // CommitSpan indicates an expected call of CommitSpan. -func (mr *MockSpannerMockRecorder) CommitSpan(ctx, heimdallSpan, state, header, chainContext interface{}) *gomock.Call { +func (mr *MockSpannerMockRecorder) CommitSpan(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitSpan", reflect.TypeOf((*MockSpanner)(nil).CommitSpan), ctx, heimdallSpan, state, header, chainContext) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitSpan", reflect.TypeOf((*MockSpanner)(nil).CommitSpan), arg0, arg1, arg2, arg3, arg4) } // GetCurrentSpan mocks base method. -func (m *MockSpanner) GetCurrentSpan(ctx context.Context, headerHash common.Hash) (*span.Span, error) { +func (m *MockSpanner) GetCurrentSpan(arg0 context.Context, arg1 common.Hash) (*span.Span, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCurrentSpan", ctx, headerHash) + ret := m.ctrl.Call(m, "GetCurrentSpan", arg0, arg1) ret0, _ := ret[0].(*span.Span) ret1, _ := ret[1].(error) return ret0, ret1 } // GetCurrentSpan indicates an expected call of GetCurrentSpan. -func (mr *MockSpannerMockRecorder) GetCurrentSpan(ctx, headerHash interface{}) *gomock.Call { +func (mr *MockSpannerMockRecorder) GetCurrentSpan(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentSpan", reflect.TypeOf((*MockSpanner)(nil).GetCurrentSpan), ctx, headerHash) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentSpan", reflect.TypeOf((*MockSpanner)(nil).GetCurrentSpan), arg0, arg1) } // GetCurrentValidatorsByBlockNrOrHash mocks base method. -func (m *MockSpanner) GetCurrentValidatorsByBlockNrOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash, blockNumber uint64) ([]*valset.Validator, error) { +func (m *MockSpanner) GetCurrentValidatorsByBlockNrOrHash(arg0 context.Context, arg1 rpc.BlockNumberOrHash, arg2 uint64) ([]*valset.Validator, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCurrentValidatorsByBlockNrOrHash", ctx, blockNrOrHash, blockNumber) + ret := m.ctrl.Call(m, "GetCurrentValidatorsByBlockNrOrHash", arg0, arg1, arg2) ret0, _ := ret[0].([]*valset.Validator) ret1, _ := ret[1].(error) return ret0, ret1 } // GetCurrentValidatorsByBlockNrOrHash indicates an expected call of GetCurrentValidatorsByBlockNrOrHash. -func (mr *MockSpannerMockRecorder) GetCurrentValidatorsByBlockNrOrHash(ctx, blockNrOrHash, blockNumber interface{}) *gomock.Call { +func (mr *MockSpannerMockRecorder) GetCurrentValidatorsByBlockNrOrHash(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentValidatorsByBlockNrOrHash", reflect.TypeOf((*MockSpanner)(nil).GetCurrentValidatorsByBlockNrOrHash), ctx, blockNrOrHash, blockNumber) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentValidatorsByBlockNrOrHash", reflect.TypeOf((*MockSpanner)(nil).GetCurrentValidatorsByBlockNrOrHash), arg0, arg1, arg2) } // GetCurrentValidatorsByHash mocks base method. -func (m *MockSpanner) GetCurrentValidatorsByHash(ctx context.Context, headerHash common.Hash, blockNumber uint64) ([]*valset.Validator, error) { +func (m *MockSpanner) GetCurrentValidatorsByHash(arg0 context.Context, arg1 common.Hash, arg2 uint64) ([]*valset.Validator, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCurrentValidatorsByHash", ctx, headerHash, blockNumber) + ret := m.ctrl.Call(m, "GetCurrentValidatorsByHash", arg0, arg1, arg2) ret0, _ := ret[0].([]*valset.Validator) ret1, _ := ret[1].(error) return ret0, ret1 } // GetCurrentValidatorsByHash indicates an expected call of GetCurrentValidatorsByHash. -func (mr *MockSpannerMockRecorder) GetCurrentValidatorsByHash(ctx, headerHash, blockNumber interface{}) *gomock.Call { +func (mr *MockSpannerMockRecorder) GetCurrentValidatorsByHash(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentValidatorsByHash", reflect.TypeOf((*MockSpanner)(nil).GetCurrentValidatorsByHash), ctx, headerHash, blockNumber) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentValidatorsByHash", reflect.TypeOf((*MockSpanner)(nil).GetCurrentValidatorsByHash), arg0, arg1, arg2) } diff --git a/eth/filters/IDatabase.go b/eth/filters/IDatabase.go index fd0824b60d..6884c6fd56 100644 --- a/eth/filters/IDatabase.go +++ b/eth/filters/IDatabase.go @@ -49,6 +49,20 @@ func (mr *MockDatabaseMockRecorder) Ancient(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ancient", reflect.TypeOf((*MockDatabase)(nil).Ancient), arg0, arg1) } +// AncientOffSet mocks base method. +func (m *MockDatabase) AncientOffSet() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AncientOffSet") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// AncientOffSet indicates an expected call of AncientOffSet. +func (mr *MockDatabaseMockRecorder) AncientOffSet() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AncientOffSet", reflect.TypeOf((*MockDatabase)(nil).AncientOffSet)) +} + // AncientRange mocks base method. func (m *MockDatabase) AncientRange(arg0 string, arg1, arg2, arg3 uint64) ([][]byte, error) { m.ctrl.T.Helper() @@ -181,6 +195,21 @@ func (mr *MockDatabaseMockRecorder) HasAncient(arg0, arg1 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasAncient", reflect.TypeOf((*MockDatabase)(nil).HasAncient), arg0, arg1) } +// ItemAmountInAncient mocks base method. +func (m *MockDatabase) ItemAmountInAncient() (uint64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ItemAmountInAncient") + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ItemAmountInAncient indicates an expected call of ItemAmountInAncient. +func (mr *MockDatabaseMockRecorder) ItemAmountInAncient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ItemAmountInAncient", reflect.TypeOf((*MockDatabase)(nil).ItemAmountInAncient)) +} + // MigrateTable mocks base method. func (m *MockDatabase) MigrateTable(arg0 string, arg1 func([]byte) ([]byte, error)) error { m.ctrl.T.Helper() From 25a775bfc428cfd31ae174cde6e69dd2157bb431 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 23 May 2023 11:21:46 +0530 Subject: [PATCH 36/53] docs/cli: update docs with inspect command --- docs/cli/README.md | 4 ++-- docs/cli/snapshot_inspect-ancient-db.md | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 docs/cli/snapshot_inspect-ancient-db.md diff --git a/docs/cli/README.md b/docs/cli/README.md index 22dbe43bed..7b80c49420 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -30,8 +30,6 @@ - [```fingerprint```](./fingerprint.md) -- [```inspect-ancient-db```](./inspect-ancient-db.md) - - [```peers```](./peers.md) - [```peers add```](./peers_add.md) @@ -48,6 +46,8 @@ - [```snapshot```](./snapshot.md) +- [```snapshot inspect-ancient-db```](./snapshot_inspect-ancient-db.md) + - [```snapshot prune-block```](./snapshot_prune-block.md) - [```snapshot prune-state```](./snapshot_prune-state.md) diff --git a/docs/cli/snapshot_inspect-ancient-db.md b/docs/cli/snapshot_inspect-ancient-db.md new file mode 100644 index 0000000000..838cfc5e9f --- /dev/null +++ b/docs/cli/snapshot_inspect-ancient-db.md @@ -0,0 +1,19 @@ +# Inspect ancient DB for block pruning + +The ```bor snapshot inspect-ancient-db``` command will inspect few fields in the ancient datastore using the given datadir location. + + +This command prints the following information which is useful for block-pruning rounds: + +1. Offset / Start block number (from kvDB). +2. Amount of items in the ancientdb. +3. Last block number written in ancientdb. + + +## Options + +- ```datadir```: Path of the data directory to store information + +- ```keystore```: Path of the data directory to store keys + +- ```datadir.ancient```: Path of the old ancient data directory From b39c7ba6fffbf6fbc084660e904c1d84a81374d3 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 11 Apr 2024 16:00:36 +0530 Subject: [PATCH 37/53] go mod tidy --- go.mod | 1 + go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 3f7497fdf9..f08ee9d83d 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml v1.9.5 github.com/peterh/liner v1.2.2 + github.com/prometheus/tsdb v0.10.0 github.com/protolambda/bls12-381-util v0.1.0 github.com/rs/cors v1.10.1 github.com/ryanuber/columnize v2.1.2+incompatible diff --git a/go.sum b/go.sum index 3e7e59986e..ed0b710764 100644 --- a/go.sum +++ b/go.sum @@ -1962,6 +1962,7 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= github.com/protolambda/bls12-381-util v0.1.0 h1:05DU2wJN7DTU7z28+Q+zejXkIsA/MF8JZQGhtBZZiWk= From afc10b6edaa72781e87381a2b93765a3f9a9e388 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 11 Apr 2024 19:08:01 +0530 Subject: [PATCH 38/53] refactor and resolve conflicts --- core/blockchain_test.go | 8 +++--- core/rawdb/chain_freezer.go | 4 +-- core/rawdb/database.go | 47 ++++++++++++++++---------------- core/rawdb/freezer.go | 33 +++++++++++++--------- core/rawdb/freezer_batch.go | 2 +- core/rawdb/freezer_resettable.go | 12 +++++++- core/rawdb/freezer_test.go | 8 +++--- core/state/pruner/pruner.go | 2 +- go.mod | 4 ++- go.sum | 2 ++ internal/cli/snapshot.go | 13 +++++++-- 11 files changed, 82 insertions(+), 53 deletions(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 78c5bda973..519c31fd28 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2017,7 +2017,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) { competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*int(defaultCacheConfig.TriesInMemory)+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false) defer db.Close() chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil, nil) @@ -2250,7 +2250,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) { }) // Import the canonical chain - diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false) defer diskdb.Close() chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil, nil) @@ -4260,7 +4260,7 @@ func testSetCanonical(t *testing.T, scheme string) { gen.AddTx(tx) }) - diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false) defer diskdb.Close() chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil, nil) @@ -4639,7 +4639,7 @@ func TestTxIndexer(t *testing.T) { for _, c := range cases { frdir := t.TempDir() - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false) _, _ = rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), append([]types.Receipts{{}}, borReceipts...), big.NewInt(0)) // Index the initial blocks from ancient store diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index ebde2687a8..4114c8a9dd 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -52,8 +52,8 @@ type chainFreezer struct { } // newChainFreezer initializes the freezer for ancient chain data. -func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFreezer, error) { - freezer, err := NewChainFreezer(datadir, namespace, readonly) +func newChainFreezer(datadir string, namespace string, readonly bool, offset uint64) (*chainFreezer, error) { + freezer, err := NewChainFreezer(datadir, namespace, readonly, offset) if err != nil { return nil, err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 290cf0d675..21c19b1f09 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -219,6 +219,17 @@ func WriteOffsetOfLastAncientFreezer(db ethdb.KeyValueWriter, offset uint64) { } } +// NewDatabaseWithOnlyFreezer create a freezer db without state +func NewDatabaseWithOnlyFreezer(db ethdb.KeyValueStore, frz, namespace string, readonly bool, newOffSet uint64) (*Freezer, error) { + // Create the idle freezer instance, this operation should be atomic to avoid mismatch between offset and acientDB. + frdb, err := NewChainFreezer(frz, namespace, readonly, newOffSet) + if err != nil { + return nil, err + } + + return frdb, nil +} + // resolveChainFreezerDir is a helper function which resolves the absolute path // of chain freezer by considering backward compatibility. func resolveChainFreezerDir(ancient string) string { @@ -243,22 +254,25 @@ func resolveChainFreezerDir(ancient string) string { return freezer } +func resolveOffset(db ethdb.KeyValueStore, isLastOffset bool) uint64 { + // The offset of ancientDB should be handled differently in different scenarios. + if isLastOffset { + return ReadOffsetOfLastAncientFreezer(db) + } else { + return ReadOffsetOfCurrentAncientFreezer(db) + } +} + // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold // storage. // //nolint:gocognit -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly, disableFreeze, isLastOffset bool) (ethdb.Database, error) { - var offset uint64 - // The offset of ancientDB should be handled differently in different scenarios. - if isLastOffset { - offset = ReadOffsetOfLastAncientFreezer(db) - } else { - offset = ReadOffsetOfCurrentAncientFreezer(db) - } +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly, disableFreeze, isLastOffset bool) (ethdb.Database, error) { + offset := resolveOffset(db, isLastOffset) // Create the idle freezer instance - frdb, err := NewFreezer(freezer, namespace, readonly, offset, freezerTableSize, FreezerNoSnappy) + frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, offset) if err != nil { return nil, err } @@ -295,7 +309,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // hasn't happened yet. If the pruning would've happened, genesis would have been wiped // from ancient db. if offset == 0 { - frgenesis, err := frdb.Ancient(freezerHashTable, 0) + frgenesis, err := frdb.Ancient(ChainFreezerHashTable, 0) if err != nil { printChainMetadata(db) return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) @@ -718,19 +732,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { logged = time.Now() } } - // Inspect append-only file store then. - ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} - for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { - if size, err := db.AncientSize(category); err == nil { - *ancientSizes[i] += common.StorageSize(size) - total += common.StorageSize(size) - } - } - // Get number of ancient rows inside the freezer - ancients := counter(0) - if count, err := db.ItemAmountInAncient(); err == nil { - ancients = counter(count) - } // Display the database statistic of key-value store. stats := [][]string{ {"Key-Value store", "Headers", headers.Size(), headers.Count()}, diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index dfa2aabf2f..21a193fef2 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -71,7 +71,7 @@ type Freezer struct { writeBatch *freezerBatch // Used during ancient db pruning - offset uint64 // Starting block number in current freezer + offset atomic.Uint64 // Starting block number in current freezer readonly bool tables map[string]*freezerTable // Data tables for storing everything @@ -79,6 +79,12 @@ type Freezer struct { closeOnce sync.Once } +// NewChainFreezer is a small utility method around NewFreezer that sets the +// default parameters for the chain storage. +func NewChainFreezer(datadir string, namespace string, readonly bool, offset uint64) (*Freezer, error) { + return NewFreezer(datadir, namespace, readonly, offset, freezerTableSize, chainFreezerNoSnappy) +} + // NewFreezer creates a freezer instance for maintaining immutable ordered // data according to the given parameters. // @@ -119,8 +125,9 @@ func NewFreezer(datadir string, namespace string, readonly bool, offset uint64, readonly: readonly, tables: make(map[string]*freezerTable), instanceLock: lock, - offset: offset, + offset: atomic.Uint64{}, } + freezer.offset.Store(offset) // Create the tables. for name, disableSnappy := range tables { @@ -153,12 +160,12 @@ func NewFreezer(datadir string, namespace string, readonly bool, offset uint64, // Some blocks in ancientDB may have already been frozen and been pruned, so adding the offset to // reprensent the absolute number of blocks already frozen. - freezer.frozen += offset + freezer.frozen.Add(offset) // Create the write batch. freezer.writeBatch = newFreezerBatch(freezer) - log.Info("Opened ancient database", "database", datadir, "readonly", readonly, "frozen", freezer.frozen, "offset", freezer.offset) + log.Info("Opened ancient database", "database", datadir, "readonly", readonly, "frozen", freezer.frozen.Load(), "offset", freezer.offset.Load()) return freezer, nil } @@ -188,7 +195,7 @@ func (f *Freezer) Close() error { // in the freezer. func (f *Freezer) HasAncient(kind string, number uint64) (bool, error) { if table := f.tables[kind]; table != nil { - return table.has(number - f.offset), nil + return table.has(number - f.offset.Load()), nil } return false, nil } @@ -196,7 +203,7 @@ func (f *Freezer) HasAncient(kind string, number uint64) (bool, error) { // Ancient retrieves an ancient binary blob from the append-only immutable files. func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) { if table := f.tables[kind]; table != nil { - return table.Retrieve(number - f.offset) + return table.Retrieve(number - f.offset.Load()) } return nil, errUnknownTable } @@ -220,13 +227,13 @@ func (f *Freezer) Ancients() (uint64, error) { } // ItemAmountInAncient returns the actual length of current ancientDB. -func (f *freezer) ItemAmountInAncient() (uint64, error) { - return atomic.LoadUint64(&f.frozen) - atomic.LoadUint64(&f.offset), nil +func (f *Freezer) ItemAmountInAncient() (uint64, error) { + return f.frozen.Load() - f.offset.Load(), nil } // AncientOffSet returns the offset of current ancientDB. -func (f *freezer) AncientOffSet() uint64 { - return atomic.LoadUint64(&f.offset) +func (f *Freezer) AncientOffSet() uint64 { + return f.offset.Load() } // Tail returns the number of first stored item in the freezer. @@ -304,7 +311,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { return oitems, nil } for _, table := range f.tables { - if err := table.truncateHead(items - f.offset); err != nil { + if err := table.truncateHead(items - f.offset.Load()); err != nil { return 0, err } } @@ -325,7 +332,7 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) { return old, nil } for _, table := range f.tables { - if err := table.truncateTail(tail - f.offset); err != nil { + if err := table.truncateTail(tail - f.offset.Load()); err != nil { return 0, err } } @@ -465,7 +472,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { return err } var ( - batch = newTable.newBatch(f.offset) + batch = newTable.newBatch(f.offset.Load()) out []byte start = time.Now() logged = time.Now() diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 04c11b7e13..fe7ae614af 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -37,7 +37,7 @@ type freezerBatch struct { func newFreezerBatch(f *Freezer) *freezerBatch { batch := &freezerBatch{tables: make(map[string]*freezerTableBatch, len(f.tables))} for kind, table := range f.tables { - batch.tables[kind] = table.newBatch(f.offset) + batch.tables[kind] = table.newBatch(f.offset.Load()) } return batch diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index 23b36a4658..c64100beea 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -54,7 +54,7 @@ func NewResettableFreezer(datadir string, namespace string, readonly bool, maxTa } opener := func() (*Freezer, error) { - return NewFreezer(datadir, namespace, readonly, maxTableSize, tables) + return NewFreezer(datadir, namespace, readonly, 0, maxTableSize, tables) } freezer, err := opener() @@ -146,6 +146,16 @@ func (f *ResettableFreezer) Ancients() (uint64, error) { return f.freezer.Ancients() } +// AncientOffSet returns the offset of current ancientDB. +func (f *ResettableFreezer) AncientOffSet() uint64 { + return f.freezer.offset.Load() +} + +// ItemAmountInAncient returns the actual length of current ancientDB. +func (f *ResettableFreezer) ItemAmountInAncient() (uint64, error) { + return f.freezer.frozen.Load() - f.freezer.offset.Load(), nil +} + // Tail returns the number of first stored item in the freezer. func (f *ResettableFreezer) Tail() (uint64, error) { f.lock.RLock() diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index d5ccb8e43f..f4b015d300 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -313,7 +313,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { // Re-openening as readonly should fail when validating // table lengths. - _, err = newFreezer(dir, "", true, 0, 2049, tables) + _, err = NewFreezer(dir, "", true, 0, 2049, tables) if err == nil { t.Fatal("readonly freezer should fail with differing table lengths") } @@ -330,7 +330,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) { t.Fatal("can't open freezer", err) } var item = make([]byte, 1024) - batch := f.tables["a"].newBatch() + batch := f.tables["a"].newBatch(0) items := uint64(10) for i := uint64(0); i < items; i++ { require.NoError(t, batch.AppendRaw(i, item)) @@ -351,7 +351,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) { go func(i int) { defer wg.Done() - f, err := NewFreezer(dir, "", true, 2049, tables) + f, err := NewFreezer(dir, "", true, 0, 2049, tables) if err == nil { fs[i] = f } else { @@ -376,7 +376,7 @@ func newFreezerForTesting(t *testing.T, tables map[string]bool) (*Freezer, strin dir := t.TempDir() // note: using low max table size here to ensure the tests actually // switch between multiple files. - f, err := NewFreezer(dir, "", false, 2049, tables) + f, err := NewFreezer(dir, "", false, 0, 2049, tables) if err != nil { t.Fatal("can't open freezer", err) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 65854f426a..0c9371612a 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -399,7 +399,7 @@ func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace str // Create new ancientDB backup and record the new and last version of offset in kvDB as well. // For every round, newoffset actually equals to the startBlockNumber in ancient backup db. - frdbBack, err := rawdb.NewFreezerDb(chainDb, p.newAncientPath, namespace, readonly, startBlockNumber) + frdbBack, err := rawdb.NewDatabaseWithOnlyFreezer(chainDb, p.newAncientPath, namespace, readonly, startBlockNumber) if err != nil { return fmt.Errorf("failed to create ancient freezer backup: %v", err) } diff --git a/go.mod b/go.mod index f08ee9d83d..76a9e6120c 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,6 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml v1.9.5 github.com/peterh/liner v1.2.2 - github.com/prometheus/tsdb v0.10.0 github.com/protolambda/bls12-381-util v0.1.0 github.com/rs/cors v1.10.1 github.com/ryanuber/columnize v2.1.2+incompatible @@ -196,6 +195,7 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/cbergoon/merkletree v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.0 // indirect github.com/etcd-io/bbolt v1.3.3 // indirect @@ -231,6 +231,7 @@ require ( github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/oapi-codegen/runtime v1.0.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/posener/complete v1.2.3 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -266,6 +267,7 @@ require ( github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae // indirect github.com/RichardKnop/machinery v1.7.4 // indirect github.com/RichardKnop/redsync v1.2.0 // indirect + github.com/prometheus/tsdb v0.10.0 github.com/zclconf/go-cty v1.13.0 // indirect github.com/zondax/hid v0.9.1 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect diff --git a/go.sum b/go.sum index ed0b710764..82c18592d6 100644 --- a/go.sum +++ b/go.sum @@ -947,6 +947,7 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -1831,6 +1832,7 @@ github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7o github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 7c54639a94..35bb202929 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -350,10 +350,17 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { if headBlock == nil { return errors.New("failed to load head block") } - headHeader := headBlock.Header() - //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) + + snapconfig := snapshot.Config{ + CacheSize: 256, + Recovery: false, + NoBuild: true, + AsyncBuild: false, + } + + // Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. + snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb, trie.HashDefaults), headBlock.Root()) if err != nil { log.Error("Unable to load snapshot", "err", err) return err // The relevant snapshot(s) might not exist From b35fea7409b2a0c37266cf2b87493957479a24f4 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 14 Apr 2024 17:39:42 +0530 Subject: [PATCH 39/53] resolve more conflicts --- cmd/geth/chaincmd.go | 2 +- cmd/geth/dbcmd.go | 41 +---------------------------- cmd/geth/snapshot.go | 4 +-- cmd/geth/verkle.go | 4 +-- cmd/utils/flags.go | 2 +- eth/state_accessor.go | 2 +- ethdb/remotedb/remotedb.go | 8 ++++++ trie/triedb/pathdb/database_test.go | 2 +- 8 files changed, 17 insertions(+), 48 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 5606216d37..b4a5db11a0 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -210,7 +210,7 @@ func initGenesis(ctx *cli.Context) error { overrides.OverrideVerkle = new(big.Int).SetInt64(v) } for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) + chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false, false, false) if err != nil { utils.Fatalf("Failed to open database: %v", err) } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 3c286d7da1..86a95e773b 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -317,7 +317,7 @@ func checkStateContent(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() var ( @@ -803,42 +803,3 @@ func showMetaData(ctx *cli.Context) error { table.Render() return nil } - -func freezerMigrate(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - db := utils.MakeChainDatabase(ctx, stack, false, false) - defer db.Close() - - // Check first block for legacy receipt format - numAncients, err := db.Ancients() - if err != nil { - return err - } - if numAncients < 1 { - log.Info("No receipts in freezer to migrate") - return nil - } - - isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db, 0) - if err != nil { - return err - } - if !isFirstLegacy { - log.Info("No legacy receipts to migrate") - return nil - } - - log.Info("Starting migration", "ancients", numAncients, "firstLegacy", firstIdx) - start := time.Now() - if err := db.MigrateTable("receipts", types.ConvertLegacyStoredReceipts); err != nil { - return err - } - if err := db.Close(); err != nil { - return err - } - log.Info("Migration finished", "duration", time.Since(start)) - - return nil -} diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 09ccabedae..dec6e4da6e 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -255,7 +255,7 @@ func checkDanglingStorage(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - db := utils.MakeChainDatabase(ctx, stack, true) + db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() return snapshot.CheckDanglingStorage(db) } @@ -688,7 +688,7 @@ func checkAccount(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) defer chaindb.Close() start := time.Now() diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index e2007c5b02..2140743d2d 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -117,7 +117,7 @@ func verifyVerkle(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) defer chaindb.Close() headBlock := rawdb.ReadHeadBlock(chaindb) @@ -174,7 +174,7 @@ func expandVerkle(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) defer chaindb.Close() var ( rootC common.Hash diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 414dc43416..9e99871c77 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2224,7 +2224,7 @@ func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database if rawdb.PreexistingDatabase(stack.ResolvePath("chaindata")) == "" { readonly = false } - return MakeChainDatabase(ctx, stack, readonly) + return MakeChainDatabase(ctx, stack, readonly, false) } func IsNetworkPreset(ctx *cli.Context) bool { diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 53525dcbec..b4180ef8c3 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -78,7 +78,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if current == nil { - return nil, fmt.Errorf("missing parent block %v %d", block.ParentHash(), block.NumberU64()-1) + return nil, nil, fmt.Errorf("missing parent block %v %d", block.ParentHash(), block.NumberU64()-1) } } else { // Otherwise, try to reexec blocks until we find a state or reach our limit diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 3003dcc144..f33d2bb9c0 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -83,6 +83,14 @@ func (db *Database) Ancients() (uint64, error) { return resp, err } +func (db *Database) AncientOffSet() uint64 { + panic("not supported") +} + +func (db *Database) ItemAmountInAncient() (uint64, error) { + panic("not supported") +} + func (db *Database) Tail() (uint64, error) { panic("not supported") } diff --git a/trie/triedb/pathdb/database_test.go b/trie/triedb/pathdb/database_test.go index 10bd50e197..f2e68d3473 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/trie/triedb/pathdb/database_test.go @@ -98,7 +98,7 @@ type tester struct { func newTester(t *testing.T, historyLimit uint64) *tester { var ( - disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false) db = New(disk, &Config{ StateHistory: historyLimit, CleanCacheSize: 256 * 1024, From 968a348f51443b1a5842a307bc72716c1632017b Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 14 Apr 2024 18:48:29 +0530 Subject: [PATCH 40/53] refactor --- internal/cli/snapshot.go | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 35bb202929..dfd35dff0f 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -389,7 +389,6 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the // entire trie. - //nolint:nestif if blob := rawdb.ReadTrieNode(chaindb, targetRoot); len(blob) == 0 { // The special case is for clique based networks(rinkeby, goerli // and some other private networks), it's possible that two @@ -402,36 +401,26 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // Note HEAD is ignored. Usually there is the associated // state available, but we don't want to use the topmost state // as the pruning target. - var found bool - for i := len(layers) - 2; i >= 1; i-- { if blob := rawdb.ReadTrieNode(chaindb, layers[i].Root()); len(blob) != 0 { targetRoot = layers[i].Root() - found = true - log.Info("Selecting middle-layer as the pruning target", "root", targetRoot, "depth", i) - - break + return nil } } - if !found { - if blob := rawdb.ReadTrieNode(chaindb, snaptree.DiskRoot()); len(blob) != 0 { - targetRoot = snaptree.DiskRoot() - found = true - - log.Info("Selecting disk-layer as the pruning target", "root", targetRoot) - } + if blob := rawdb.ReadTrieNode(chaindb, snaptree.DiskRoot()); len(blob) != 0 { + targetRoot = snaptree.DiskRoot() + log.Info("Selecting disk-layer as the pruning target", "root", targetRoot) + return nil } - if !found { - if len(layers) > 0 { - log.Error("no snapshot paired state") - return errors.New("no snapshot paired state") - } - - return fmt.Errorf("associated state[%x] is not present", targetRoot) + if len(layers) > 0 { + log.Error("no snapshot paired state") + return errors.New("no snapshot paired state") } + + return fmt.Errorf("associated state[%x] is not present", targetRoot) } else { if len(layers) > 0 { log.Info("Selecting bottom-most difflayer as the pruning target", "root", targetRoot, "height", headHeader.Number.Uint64()-uint64(len(layers)-1)) From 8dbde87d6dc9774abeaa69d4deb2084197ebe6d0 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 14 Apr 2024 18:51:36 +0530 Subject: [PATCH 41/53] explicitly read node for hash scheme --- internal/cli/snapshot.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index dfd35dff0f..f8f73a7e95 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -389,7 +389,7 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the // entire trie. - if blob := rawdb.ReadTrieNode(chaindb, targetRoot); len(blob) == 0 { + if blob := rawdb.ReadTrieNode(chaindb, common.Hash{}, nil, targetRoot, rawdb.HashScheme); len(blob) == 0 { // The special case is for clique based networks(rinkeby, goerli // and some other private networks), it's possible that two // consecutive blocks will have same root. In this case snapshot @@ -402,14 +402,14 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { // state available, but we don't want to use the topmost state // as the pruning target. for i := len(layers) - 2; i >= 1; i-- { - if blob := rawdb.ReadTrieNode(chaindb, layers[i].Root()); len(blob) != 0 { + if blob := rawdb.ReadTrieNode(chaindb, common.Hash{}, nil, layers[i].Root(), rawdb.HashScheme); len(blob) != 0 { targetRoot = layers[i].Root() log.Info("Selecting middle-layer as the pruning target", "root", targetRoot, "depth", i) return nil } } - if blob := rawdb.ReadTrieNode(chaindb, snaptree.DiskRoot()); len(blob) != 0 { + if blob := rawdb.ReadTrieNode(chaindb, common.Hash{}, nil, snaptree.DiskRoot(), rawdb.HashScheme); len(blob) != 0 { targetRoot = snaptree.DiskRoot() log.Info("Selecting disk-layer as the pruning target", "root", targetRoot) return nil From ac681be6b09a18e2ff30cb3a455ac576bae737bb Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 14 Apr 2024 19:07:45 +0530 Subject: [PATCH 42/53] add check for hash scheme, fix tests --- internal/cli/snapshot.go | 6 ++++++ internal/cli/snapshot_test.go | 28 ++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index f8f73a7e95..a6a11277bb 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -346,6 +346,12 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { return nil } + // Check if we're in hash scheme and not path scheme + if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { + log.Warn("Snapshot and MPT check is only supported for hash scheme, skipping") + return nil + } + headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { return errors.New("failed to load head block") diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index 880dcc0da1..d17893ddda 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/stretchr/testify/require" ) @@ -79,7 +80,17 @@ func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64 err = testBlockPruner.BlockPruneBackup(chaindbPath, 512, dbHandles, "", false, false) require.NoError(t, err, "failed to backup block") - dbBack, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, newAncientPath, "", false, true, false) + dbBack, err := rawdb.Open(rawdb.OpenOptions{ + Type: node.Config().DBEngine, + Directory: chaindbPath, + AncientsDirectory: newAncientPath, + Namespace: "", + Cache: 0, + Handles: 0, + ReadOnly: false, + DisableFreeze: true, + IsLastOffset: false, + }) require.NoError(t, err, "failed to create db with ancient backend") defer dbBack.Close() @@ -133,14 +144,23 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai t.Helper() // Create a database with ancient freezer - db, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, AncientPath, "", false, false, false) + db, err := rawdb.Open(rawdb.OpenOptions{ + Directory: chaindbPath, + AncientsDirectory: AncientPath, + Namespace: "", + Cache: 0, + Handles: 0, + ReadOnly: false, + DisableFreeze: false, + IsLastOffset: false, + }) require.NoError(t, err, "failed to create db with ancient backend") defer db.Close() - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) // Initialize a fresh chain with only a genesis block - blockchain, err := core.NewBlockChain(db, config, gspec.Config, engine, vm.Config{}, nil, nil, nil) + blockchain, err := core.NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil, nil) require.NoError(t, err, "failed to create chain") // Make chain starting from genesis From 99f684f768b0090c85cdaeff7bf7c6731f9caffe Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Sun, 14 Apr 2024 19:13:44 +0530 Subject: [PATCH 43/53] fix typo --- core/rawdb/freezer_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 214f0f948a..0e42e31017 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -1069,7 +1069,7 @@ func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { func (t *freezerTable) Fill(number uint64) error { if t.items.Load() < number { b := t.newBatch(0) - log.Info("Filling all data into freezer for backward compatablity", "name", t.name, "items", &t.items, "number", number) + log.Info("Filling all data into freezer for backward compatibility", "name", t.name, "items", &t.items, "number", number) for t.items.Load() < number { if err := b.Append(t.items.Load(), nil); err != nil { From a6aa79a5ba1871540ac845cc5fe3c54046c91e36 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 15 Apr 2024 13:31:52 +0530 Subject: [PATCH 44/53] update docs and add warning --- docs/cli/server.md | 6 ++++-- docs/cli/snapshot_inspect-ancient-db.md | 4 ++-- docs/cli/snapshot_prune-block.md | 12 ++++++------ docs/cli/snapshot_prune-state.md | 8 +++++++- internal/cli/snapshot.go | 10 ++++++++-- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/cli/server.md b/docs/cli/server.md index 18f72d5993..fbd4adad3f 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -20,7 +20,7 @@ The ```bor server``` command runs the Bor client. - ```bor.withoutheimdall```: Run without Heimdall service (for testing purpose) (default: false) -- ```chain```: Name of the chain to sync ('mumbai', 'mainnet', 'amoy') or path to a genesis file (default: mainnet) +- ```chain```: Name of the chain to sync ('amoy', 'mumbai', 'mainnet') or path to a genesis file (default: mainnet) - ```config```: Path to the TOML configuration file @@ -84,6 +84,8 @@ The ```bor server``` command runs the Bor client. - ```snapshot```: Enables the snapshot-database mode (default: true) +- ```state.scheme```: Scheme to use for storing ethereum state ('hash' or 'path') (default: hash) + - ```syncmode```: Blockchain sync mode (only "full" sync supported) (default: full) - ```verbosity```: Logging verbosity for the server (5=trace|4=debug|3=info|2=warn|1=error|0=crit) (default: 3) @@ -306,4 +308,4 @@ The ```bor server``` command runs the Bor client. - ```txpool.pricelimit```: Minimum gas price limit to enforce for acceptance into the pool (default: 1) -- ```txpool.rejournal```: Time interval to regenerate the local transaction journal (default: 1h0m0s) +- ```txpool.rejournal```: Time interval to regenerate the local transaction journal (default: 1h0m0s) \ No newline at end of file diff --git a/docs/cli/snapshot_inspect-ancient-db.md b/docs/cli/snapshot_inspect-ancient-db.md index 838cfc5e9f..c5742e75c1 100644 --- a/docs/cli/snapshot_inspect-ancient-db.md +++ b/docs/cli/snapshot_inspect-ancient-db.md @@ -14,6 +14,6 @@ This command prints the following information which is useful for block-pruning - ```datadir```: Path of the data directory to store information -- ```keystore```: Path of the data directory to store keys - - ```datadir.ancient```: Path of the old ancient data directory + +- ```keystore```: Path of the data directory to store keys \ No newline at end of file diff --git a/docs/cli/snapshot_prune-block.md b/docs/cli/snapshot_prune-block.md index b966cb5e0f..de663386e4 100644 --- a/docs/cli/snapshot_prune-block.md +++ b/docs/cli/snapshot_prune-block.md @@ -13,18 +13,18 @@ The purpose of doing it is because the block data will be moved into the ancient ## Options -- ```datadir```: Path of the data directory to store information - -- ```keystore```: Path of the data directory to store keys - -- ```datadir.ancient```: Path of the old ancient data directory - - ```block-amount-reserved```: Sets the expected reserved number of blocks for offline block prune (default: 1024) - ```cache.triesinmemory```: Number of block states (tries) to keep in memory (default = 128) (default: 128) - ```check-snapshot-with-mpt```: Enable checking between snapshot and MPT (default: false) +- ```datadir```: Path of the data directory to store information + +- ```datadir.ancient```: Path of the old ancient data directory + +- ```keystore```: Path of the data directory to store keys + ### Cache Options - ```cache```: Megabytes of memory allocated to internal caching (default: 1024) \ No newline at end of file diff --git a/docs/cli/snapshot_prune-state.md b/docs/cli/snapshot_prune-state.md index bd053324da..1d13c379d5 100644 --- a/docs/cli/snapshot_prune-state.md +++ b/docs/cli/snapshot_prune-state.md @@ -1,6 +1,12 @@ # Prune state -The ```bor snapshot prune-state``` command will prune historical state data with the help of the state snapshot. All trie nodes and contract codes that do not belong to the specified version state will be deleted from the database. After pruning, only two version states are available: genesis and the specific one. +The ```bor snapshot prune-state``` command will prune historical state data +with the help of the state snapshot. All trie nodes and contract codes that do not belong to the +specified version state will be deleted from the database. After pruning, only two version states +are available: genesis and the specific one. + +Warning: This command only works with hash based storage scheme and doesn't work with path based +storage scheme. ## Options diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index a6a11277bb..648df1fe31 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -84,8 +84,14 @@ type PruneStateCommand struct { // MarkDown implements cli.MarkDown interface func (c *PruneStateCommand) MarkDown() string { items := []string{ - "# Prune state", - "The ```bor snapshot prune-state``` command will prune historical state data with the help of the state snapshot. All trie nodes and contract codes that do not belong to the specified version state will be deleted from the database. After pruning, only two version states are available: genesis and the specific one.", + `# Prune state`, + `The ` + "```" + "bor snapshot prune-state" + "```" + ` command will prune historical state data +with the help of the state snapshot. All trie nodes and contract codes that do not belong to the +specified version state will be deleted from the database. After pruning, only two version states +are available: genesis and the specific one. + +Warning: This command only works with hash based storage scheme and doesn't work with path based +storage scheme.`, c.Flags().MarkDown(), } From 0f1b9e2d5a00a850a200e088681663f45f0cf9c1 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 16 Apr 2024 10:31:19 +0530 Subject: [PATCH 45/53] raise error if pbss is enabled --- docs/cli/snapshot_prune-block.md | 4 +++- docs/cli/snapshot_prune-state.md | 3 --- internal/cli/snapshot.go | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/cli/snapshot_prune-block.md b/docs/cli/snapshot_prune-block.md index de663386e4..b3ae3d9d4f 100644 --- a/docs/cli/snapshot_prune-block.md +++ b/docs/cli/snapshot_prune-block.md @@ -9,7 +9,9 @@ The brief workflow as below: 2. then delete the original ancientdb dir and rename the ancient_backup to original one for replacement, 3. finally assemble the statedb and new ancientdb together. -The purpose of doing it is because the block data will be moved into the ancient store when it becomes old enough(exceed the Threshold 90000), the disk usage will be very large over time, and is occupied mainly by ancientdb, so it's very necessary to do block data pruning, this feature will handle it. +The purpose of doing it is because the block data will be moved into the ancient store when it becomes old enough (exceed the Threshold 90000), the disk usage will be very large over time, and is occupied mainly by ancientdb, so it's very necessary to do block data pruning, this feature will handle it. + +Warning: This command only works with hash based storage scheme and doesn't work with path based storage scheme. ## Options diff --git a/docs/cli/snapshot_prune-state.md b/docs/cli/snapshot_prune-state.md index 1d13c379d5..ef955e8af0 100644 --- a/docs/cli/snapshot_prune-state.md +++ b/docs/cli/snapshot_prune-state.md @@ -5,9 +5,6 @@ with the help of the state snapshot. All trie nodes and contract codes that do n specified version state will be deleted from the database. After pruning, only two version states are available: genesis and the specific one. -Warning: This command only works with hash based storage scheme and doesn't work with path based -storage scheme. - ## Options - ```bloomfilter.size```: Size of the bloom filter (default: 2048) diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 648df1fe31..aeb47c18e0 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -24,6 +24,8 @@ import ( "github.com/mitchellh/cli" ) +var errPbssNotSupported = errors.New("ancient block pruning is not supporeted on path based storage scheme") + // SnapshotCommand is the command to group the snapshot commands type SnapshotCommand struct { UI cli.Ui @@ -88,10 +90,7 @@ func (c *PruneStateCommand) MarkDown() string { `The ` + "```" + "bor snapshot prune-state" + "```" + ` command will prune historical state data with the help of the state snapshot. All trie nodes and contract codes that do not belong to the specified version state will be deleted from the database. After pruning, only two version states -are available: genesis and the specific one. - -Warning: This command only works with hash based storage scheme and doesn't work with path based -storage scheme.`, +are available: genesis and the specific one.`, c.Flags().MarkDown(), } @@ -234,7 +233,9 @@ The brief workflow as below: 2. then delete the original ancientdb dir and rename the ancient_backup to original one for replacement, 3. finally assemble the statedb and new ancientdb together. -The purpose of doing it is because the block data will be moved into the ancient store when it becomes old enough(exceed the Threshold 90000), the disk usage will be very large over time, and is occupied mainly by ancientdb, so it's very necessary to do block data pruning, this feature will handle it.`, +The purpose of doing it is because the block data will be moved into the ancient store when it becomes old enough (exceed the Threshold 90000), the disk usage will be very large over time, and is occupied mainly by ancientdb, so it's very necessary to do block data pruning, this feature will handle it. + +Warning: This command only works with hash based storage scheme and doesn't work with path based storage scheme.`, c.Flags().MarkDown(), } @@ -348,13 +349,12 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { } defer chaindb.Close() - if !c.checkSnapshotWithMPT { - return nil + // Check if we're using hash based scheme and not path based + if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { + return errPbssNotSupported } - // Check if we're in hash scheme and not path scheme - if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { - log.Warn("Snapshot and MPT check is only supported for hash scheme, skipping") + if !c.checkSnapshotWithMPT { return nil } From aba60911633d06758830ae4bfda56d1bc4101155 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 16 Apr 2024 16:40:49 +0530 Subject: [PATCH 46/53] revert read raw bor receipt change --- core/rawdb/bor_receipt.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/core/rawdb/bor_receipt.go b/core/rawdb/bor_receipt.go index b8226f98cb..a27b8e3b9a 100644 --- a/core/rawdb/bor_receipt.go +++ b/core/rawdb/bor_receipt.go @@ -68,18 +68,8 @@ func ReadRawBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types. // Convert the receipts from their storage form to their internal representation var storageReceipt types.ReceiptForStorage if err := rlp.DecodeBytes(data, &storageReceipt); err != nil { - storageReceipts := []*types.ReceiptForStorage{} - if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { - log.Error("Invalid bor receipt array RLP", "number", number, "hash", hash, "err", err) - return nil - } - - if nReceipts := len(storageReceipts); nReceipts != 1 { - log.Error("Invalid bor receipt array RLP length", "number", number, "hash", hash, "nReceipts", nReceipts) - return nil - } - - return (*types.Receipt)(storageReceipts[0]) + log.Error("Invalid bor receipt RLP", "hash", hash, "err", err) + return nil } return (*types.Receipt)(&storageReceipt) From 15c806e4d7d67fd33225703a53f3c7ac4f6e5d75 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 22 Apr 2024 16:11:28 +0530 Subject: [PATCH 47/53] consensus/bor: handle nil header case in get root hash --- consensus/bor/api.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/bor/api.go b/consensus/bor/api.go index 6d72e309e3..0de1ed9cab 100644 --- a/consensus/bor/api.go +++ b/consensus/bor/api.go @@ -317,6 +317,10 @@ func (api *API) GetRootHash(start uint64, end uint64) (string, error) { for i := 0; i < len(blockHeaders); i++ { blockHeader := blockHeaders[i] + // Handle no header case, which is possible if ancient pruning was done + if blockHeader == nil { + return "", errUnknownBlock + } header := crypto.Keccak256(appendBytes32( blockHeader.Number.Bytes(), new(big.Int).SetUint64(blockHeader.Time).Bytes(), From 0c49dfc64414548f4a50e039ccd9128d101eb791 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 22 Apr 2024 17:14:49 +0530 Subject: [PATCH 48/53] address comments --- cmd/utils/flags.go | 2 +- core/rawdb/database.go | 8 ++++++-- core/rawdb/freezer.go | 1 - core/rawdb/schema.go | 4 ++-- core/state/pruner/pruner.go | 5 +++++ internal/cli/snapshot.go | 8 ++++++-- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 9e99871c77..0e5ecc2b52 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2183,7 +2183,7 @@ func SplitTagsFlag(tagsFlag string) map[string]string { return tagsMap } -// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. +// MakeChainDatabase opens a LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database { var ( cache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 21c19b1f09..96086568e6 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -138,11 +138,12 @@ func (db *nofreezedb) TruncateTail(items uint64) (uint64, error) { return 0, errNotSupported } -// Ancients returns an error as we don't have a backing chain freezer. +// ItemAmountInAncient returns an error as we don't have a backing chain freezer. func (db *nofreezedb) ItemAmountInAncient() (uint64, error) { return 0, errNotSupported } +// AncientOffSet returns 0 as we don't have a backing chain freezer. func (db *nofreezedb) AncientOffSet() uint64 { return 0 } @@ -254,6 +255,8 @@ func resolveChainFreezerDir(ancient string) string { return freezer } +// resolveOffset is a helper function which resolves the value of offset to use +// while opening a chain freezer. func resolveOffset(db ethdb.KeyValueStore, isLastOffset bool) uint64 { // The offset of ancientDB should be handled differently in different scenarios. if isLastOffset { @@ -340,7 +343,6 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st } // We are about to exit on error. Print database metadata before exiting printChainMetadata(db) - // return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", startBlock) return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d]", startBlock-1, number, head) } @@ -562,6 +564,8 @@ func (s *stat) Count() string { return s.count.String() } +// AncientInspect inspects the underlying freezer db and prints stats relevant +// for ancient data pruning. It prints the start and end blocks of freezer db. func AncientInspect(db ethdb.Database) error { offset := counter(ReadOffsetOfCurrentAncientFreezer(db)) // Get number of ancient rows inside the freezer. diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 21a193fef2..eae6705b68 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -125,7 +125,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, offset uint64, readonly: readonly, tables: make(map[string]*freezerTable), instanceLock: lock, - offset: atomic.Uint64{}, } freezer.offset.Store(offset) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3406fe1b0d..0f10b6d33a 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -83,10 +83,10 @@ var ( fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") // offset of the new updated ancientDB. - offsetOfCurrentAncientFreezer = []byte("offSetOfCurrentAncientFreezer") + offsetOfCurrentAncientFreezer = []byte("offsetOfCurrentAncientFreezer") // offset of the ancientDB before updated version. - offsetOfLastAncientFreezer = []byte("offSetOfLastAncientFreezer") + offsetOfLastAncientFreezer = []byte("offsetOfLastAncientFreezer") // badBlockKey tracks the list of bad blocks seen by local badBlockKey = []byte("InvalidBlock") diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 0c9371612a..abbc0044a6 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -352,6 +352,8 @@ func NewBlockPruner(n *node.Node, oldAncientPath, newAncientPath string, BlockAm } } +// backupOldDb takes a backup of the old ancient db to a new ancient db. It moves the contents based on the +// value of number of blocks to be reserved. func (p *BlockPruner) backupOldDb(name string, cache, handles int, namespace string, readonly, interrupt bool) error { log.Info("Backup old ancientDB", "oldAncientPath", p.oldAncientPath, "newAncientPath", p.newAncientPath) // Open old db wrapper. @@ -482,6 +484,7 @@ func (p *BlockPruner) BlockPruneBackup(name string, cache, handles int, namespac return nil } +// RecoverInterruption handles the case when the block prune process was interrupted. func (p *BlockPruner) RecoverInterruption(name string, cache, handles int, namespace string, readonly bool) error { log.Info("RecoverInterruption for block prune") @@ -548,6 +551,8 @@ func checkFileExist(path string) (bool, error) { return true, nil } +// AncientDbReplacer deletes the old ancient db and points the new db +// to the old path. func (p *BlockPruner) AncientDbReplacer() error { // Delete directly for the old ancientDB, e.g.: path ../chaindb/ancient if err := os.RemoveAll(p.oldAncientPath); err != nil { diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index aeb47c18e0..a575210bb5 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -327,7 +327,7 @@ func (c *PruneBlockCommand) Run(args []string) int { return 1 } - err = c.accessDb(node, dbHandles) + err = c.validateAgainstSnapshot(node, dbHandles) if err != nil { c.UI.Error(err.Error()) return 1 @@ -342,7 +342,8 @@ func (c *PruneBlockCommand) Run(args []string) int { return 0 } -func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { +// validateAgainstSnapshot checks if the MPT data and snapshot data matches with each other or not +func (c *PruneBlockCommand) validateAgainstSnapshot(stack *node.Node, dbHandles int) error { chaindb, err := stack.OpenDatabaseWithFreezer(chaindataPath, c.cache, dbHandles, c.datadirAncient, "", false, true, false) if err != nil { return fmt.Errorf("failed to accessdb %v", err) @@ -444,6 +445,9 @@ func (c *PruneBlockCommand) accessDb(stack *node.Node, dbHandles int) error { return nil } +// pruneBlock is the entry point for the ancient pruning process. Based on the user specified +// params, it will prune the ancient data. It also handles the case where the pruning process +// was interrupted earlier. func (c *PruneBlockCommand) pruneBlock(stack *node.Node, fdHandles int) error { name := "chaindata" From 3c38710bb41af39281c69cc941e7a14714dc6714 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 23 Apr 2024 11:48:47 +0530 Subject: [PATCH 49/53] core/rawdb: check chain continuity by matching parent hash --- core/rawdb/accessors_chain.go | 17 +++++++++++++++++ core/rawdb/database.go | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index aeed8da55c..bfea75610b 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -165,6 +165,23 @@ func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) *uint64 { return &number } +// ReadHeaderFromKvStore retrieves the block header corresponding to the hash only +// from the underlying key-value store and doesn't use ancient freezer for backup. +func ReadHeaderFromKvStore(db ethdb.KeyValueReader, hash common.Hash, number uint64) *types.Header { + data, _ := db.Get(headerKey(number, hash)) + if len(data) == 0 { + return nil + } + + header := new(types.Header) + if err := rlp.DecodeBytes(data, header); err != nil { + log.Error("Invalid block header RLP", "hash", hash, "err", err) + return nil + } + + return header +} + // WriteHeaderNumber stores the hash->number mapping. func WriteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { key := headerNumberKey(hash) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 96086568e6..70f05ec88a 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -329,7 +329,8 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // be less and hence we need to calculate the first block of leveldb by adding // the offset to it i.e. start block of leveldb = frozen + offset. startBlock := frozen + offset - if kvhash, _ := db.Get(headerHashKey(startBlock)); len(kvhash) == 0 { + var kvhash []byte + if kvhash, _ = db.Get(headerHashKey(startBlock)); len(kvhash) == 0 { // Subsequent header after the freezer limit is missing from the database. // Reject startup if the database has a more recent head. if head := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); head > startBlock-1 { @@ -351,7 +352,21 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st } // Otherwise, key-value store continues where the freezer left off, all is fine. // We might have duplicate blocks (crash after freezer write but before key-value - // store deletion, but that's fine). + // store deletion, but that's fine). Still, check if the first block of key-value + // store points to last block in freezer. + if head := ReadHeaderFromKvStore(db, common.BytesToHash(kvhash), startBlock); head != nil { + parentHash := head.ParentHash.Bytes() + ancientParentHash, _ := frdb.Ancient(ChainFreezerHashTable, startBlock-1) + if ancientParentHash == nil { + printChainMetadata(db) + return nil, fmt.Errorf("missing parent hash for block #%d in ancient", startBlock-1) + } + if !bytes.Equal(parentHash, ancientParentHash) { + printChainMetadata(db) + return nil, fmt.Errorf("broken chain due to parent hash mismatch: %#x (leveldb) != %#x (ancients) for block #%d, please set --datadir.ancient to the correct path", parentHash, ancientParentHash, startBlock-1) + } + // First block of key-value store points back to correct parent in ancient + } } else { // This case means the freezer is empty. Either nothing is moved from the // key-value store or we've pruned all data. From 8e4b0bd1bc3f5d8c6d08416835c2425a631e646d Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 23 Apr 2024 12:03:45 +0530 Subject: [PATCH 50/53] core/rawdb: account for pruned ancient blocks --- core/rawdb/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 70f05ec88a..3762a4366f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -304,7 +304,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st // it to the freezer content. //nolint:nestif if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 { - if frozen, _ := frdb.Ancients(); frozen > 0 { + if frozen, _ := frdb.ItemAmountInAncient(); frozen > 0 { // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. From dbe34496415b93938535e0b2c6857530e6422daf Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 8 May 2024 16:51:59 +0530 Subject: [PATCH 51/53] go mod tidy --- go.mod | 2 -- go.sum | 2 -- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index 01d3fbe78b..6c4ee2c49f 100644 --- a/go.mod +++ b/go.mod @@ -195,7 +195,6 @@ require ( github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/cbergoon/merkletree v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.0 // indirect github.com/etcd-io/bbolt v1.3.3 // indirect @@ -231,7 +230,6 @@ require ( github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/oapi-codegen/runtime v1.0.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect github.com/posener/complete v1.2.3 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect diff --git a/go.sum b/go.sum index 1d314dbd0e..aee1527e94 100644 --- a/go.sum +++ b/go.sum @@ -956,7 +956,6 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -1826,7 +1825,6 @@ github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7o github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= From 55b535745ebb17fa1c0f632f0b5281275d526122 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 8 May 2024 16:59:30 +0530 Subject: [PATCH 52/53] fix tests --- cmd/geth/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 1cd356d0f9..05345120da 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -683,7 +683,7 @@ func snapshotExportPreimages(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chaindb := utils.MakeChainDatabase(ctx, stack, true) + chaindb := utils.MakeChainDatabase(ctx, stack, true, false) defer chaindb.Close() triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false) From d098c599e894c3ea7c11f01b823ad0cc2cf406da Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 8 May 2024 19:18:15 +0530 Subject: [PATCH 53/53] fix tests --- internal/ethapi/api.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 54baf0dea6..9c706c844f 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -684,10 +684,6 @@ func (s *BlockChainAPI) GetTransactionReceiptsByBlock(ctx context.Context, block return nil, err } - if receipts == nil { - return nil, fmt.Errorf("block %d receipts not found", block.NumberU64()) - } - txs := block.Transactions() var txHash common.Hash