-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[R4R]offline block prune #543
Merged
+879
−65
Merged
Changes from 23 commits
Commits
Show all changes
26 commits
Select commit
Hold shift + click to select a range
e42f4e3
offline block prune
Mercybudda 9271890
update
Mercybudda d4d8324
update
Mercybudda 021aba5
update and add unit test
Mercybudda 8790427
addressed comments from walt
Mercybudda fccd50e
Addressed comments from walt and Igor
Mercybudda be04eba
ensure MPT and snapshot matched
Mercybudda cb1ca71
add one more parameter to indicate blockprune
Mercybudda 7b662d5
update the logic of creating freezerDb
Mercybudda eb1263d
update flag command description
Mercybudda d1fb290
expose the function for db inspect the offset/startBlockNumber
Mercybudda f532e37
add flags to inspect prune info
Mercybudda c14e873
rename flag of reserved-recent-blocks to block-amount-reserved
Mercybudda aaaee6b
addressed comments from walt
Mercybudda a2ed56c
handle the case of command interruption
Mercybudda 6b4031a
refined goimports
Mercybudda 8e051d4
addressed comments from walt
Mercybudda a516065
change the logic as restarting prune after interruption
Mercybudda de86417
addressed comments
Mercybudda f3e31a1
reclaimed freezer logic
Mercybudda 69cdcfe
introduce flag to enable/disable check between MPT and snapshot
Mercybudda f9ea6a2
update the logic of frozen field in freezerDB
Mercybudda 6e13383
update the code in all places related to freezer change
Mercybudda 29279f2
addressed comments from dylan
Mercybudda dbfc231
update the logic for backup block difficulty
Mercybudda 37f2e89
addressed comments from dylan
Mercybudda File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -48,3 +48,5 @@ profile.cov | |
| /dashboard/assets/package-lock.json | ||
|
|
||
| **/yarn-error.log | ||
| cmd/geth/node/ | ||
| cmd/geth/__debug_bin | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,241 @@ | ||
| // Copyright 2016 The go-ethereum Authors | ||
| // This file is part of go-ethereum. | ||
| // | ||
| // go-ethereum is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // go-ethereum is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU General Public License | ||
| // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/hex" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "math/big" | ||
| "os" | ||
| "path/filepath" | ||
| "reflect" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/ethereum/go-ethereum/cmd/utils" | ||
| "github.com/ethereum/go-ethereum/common" | ||
| "github.com/ethereum/go-ethereum/consensus" | ||
| "github.com/ethereum/go-ethereum/consensus/ethash" | ||
| "github.com/ethereum/go-ethereum/core" | ||
| "github.com/ethereum/go-ethereum/core/rawdb" | ||
| "github.com/ethereum/go-ethereum/core/state/pruner" | ||
| "github.com/ethereum/go-ethereum/core/types" | ||
| "github.com/ethereum/go-ethereum/core/vm" | ||
| "github.com/ethereum/go-ethereum/crypto" | ||
| "github.com/ethereum/go-ethereum/eth" | ||
| "github.com/ethereum/go-ethereum/ethdb" | ||
| "github.com/ethereum/go-ethereum/node" | ||
| "github.com/ethereum/go-ethereum/params" | ||
| "github.com/ethereum/go-ethereum/rlp" | ||
| ) | ||
|
|
||
| var ( | ||
| canonicalSeed = 1 | ||
| blockPruneBackUpBlockNumber = 128 | ||
| key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") | ||
| address = crypto.PubkeyToAddress(key.PublicKey) | ||
| balance = big.NewInt(10000000) | ||
| gspec = &core.Genesis{Config: params.TestChainConfig, Alloc: core.GenesisAlloc{address: {Balance: balance}}} | ||
| signer = types.LatestSigner(gspec.Config) | ||
| config = &core.CacheConfig{ | ||
| TrieCleanLimit: 256, | ||
| TrieDirtyLimit: 256, | ||
| TrieTimeLimit: 5 * time.Minute, | ||
| SnapshotLimit: 0, // Disable snapshot | ||
| TriesInMemory: 128, | ||
| } | ||
| engine = ethash.NewFullFaker() | ||
| ) | ||
|
|
||
| func TestOfflineBlockPrune(t *testing.T) { | ||
| //Corner case for 0 remain in ancinetStore. | ||
| testOfflineBlockPruneWithAmountReserved(t, 0) | ||
| //General case. | ||
| testOfflineBlockPruneWithAmountReserved(t, 100) | ||
| } | ||
|
|
||
| func testOfflineBlockPruneWithAmountReserved(t *testing.T, amountReserved uint64) { | ||
| datadir, err := ioutil.TempDir("", "") | ||
| if err != nil { | ||
| t.Fatalf("Failed to create temporary datadir: %v", err) | ||
| } | ||
| os.RemoveAll(datadir) | ||
|
|
||
| chaindbPath := filepath.Join(datadir, "chaindata") | ||
| oldAncientPath := filepath.Join(chaindbPath, "ancient") | ||
| newAncientPath := filepath.Join(chaindbPath, "ancient_back") | ||
|
|
||
| db, blocks, blockList, receiptsList, externTdList, startBlockNumber, _ := BlockchainCreator(t, chaindbPath, oldAncientPath, amountReserved) | ||
| node, _ := startEthService(t, gspec, blocks, chaindbPath) | ||
| defer node.Close() | ||
|
|
||
| //Initialize a block pruner for pruning, only remain amountReserved blocks backward. | ||
| testBlockPruner, err := pruner.NewBlockPruner(db, node, oldAncientPath, newAncientPath, amountReserved) | ||
| if err != nil { | ||
| t.Fatalf("failed to make new blockpruner: %v", err) | ||
| } | ||
| if err := testBlockPruner.BlockPruneBackUp(chaindbPath, 512, utils.MakeDatabaseHandles(), "", false, false); err != nil { | ||
| t.Fatalf("Failed to back up block: %v", err) | ||
| } | ||
|
|
||
| dbBack, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, newAncientPath, "", false, true, false) | ||
| if err != nil { | ||
| t.Fatalf("failed to create database with ancient backend") | ||
| } | ||
| defer dbBack.Close() | ||
|
|
||
| //check against if the backup data matched original one | ||
| for blockNumber := startBlockNumber; blockNumber < startBlockNumber+amountReserved; blockNumber++ { | ||
| blockHash := rawdb.ReadCanonicalHash(dbBack, blockNumber) | ||
| block := rawdb.ReadBlock(dbBack, blockHash, blockNumber) | ||
| if reflect.DeepEqual(block, blockList[blockNumber-startBlockNumber]) { | ||
| t.Fatalf("block data did not match between oldDb and backupDb") | ||
| } | ||
|
|
||
| receipts := rawdb.ReadRawReceipts(dbBack, blockHash, blockNumber) | ||
| if err := checkReceiptsRLP(receipts, receiptsList[blockNumber-startBlockNumber]); err != nil { | ||
| t.Fatalf("receipts did not match between oldDb and backupDb") | ||
| } | ||
| // // Calculate the total difficulty of the block | ||
| td := rawdb.ReadTd(dbBack, blockHash, blockNumber) | ||
| if td == nil { | ||
| t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) | ||
| } | ||
| externTd := new(big.Int).Add(block.Difficulty(), td) | ||
| if reflect.DeepEqual(externTd, externTdList[blockNumber-startBlockNumber]) { | ||
| t.Fatalf("externTd did not match between oldDb and backupDb") | ||
| } | ||
| } | ||
|
|
||
| //check if ancientDb freezer replaced successfully | ||
| testBlockPruner.AncientDbReplacer() | ||
| if _, err := os.Stat(newAncientPath); err != nil { | ||
| if !os.IsNotExist(err) { | ||
| t.Fatalf("ancientDb replaced unsuccessfully") | ||
| } | ||
| } | ||
| if _, err := os.Stat(oldAncientPath); err != nil { | ||
| t.Fatalf("ancientDb replaced unsuccessfully") | ||
| } | ||
| } | ||
|
|
||
| func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemain uint64) (ethdb.Database, []*types.Block, []*types.Block, []types.Receipts, []*big.Int, uint64, *core.BlockChain) { | ||
| //create a database with ancient freezer | ||
| db, err := rawdb.NewLevelDBDatabaseWithFreezer(chaindbPath, 0, 0, AncientPath, "", false, false, false) | ||
| if err != nil { | ||
| t.Fatalf("failed to create database with ancient backend") | ||
| } | ||
| defer db.Close() | ||
| genesis := gspec.MustCommit(db) | ||
| // Initialize a fresh chain with only a genesis block | ||
| blockchain, err := core.NewBlockChain(db, config, gspec.Config, engine, vm.Config{}, nil, nil) | ||
| if err != nil { | ||
| t.Fatalf("Failed to create chain: %v", err) | ||
| } | ||
|
|
||
| // Make chain starting from genesis | ||
| blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 500, func(i int, block *core.BlockGen) { | ||
| block.SetCoinbase(common.Address{0: byte(canonicalSeed), 19: byte(i)}) | ||
| tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, nil, nil), signer, key) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| block.AddTx(tx) | ||
| block.SetDifficulty(big.NewInt(1000000)) | ||
| }) | ||
| if _, err := blockchain.InsertChain(blocks); err != nil { | ||
| t.Fatalf("Failed to import canonical chain start: %v", err) | ||
| } | ||
|
|
||
| // Force run a freeze cycle | ||
| type freezer interface { | ||
| Freeze(threshold uint64) error | ||
| Ancients() (uint64, error) | ||
| } | ||
| db.(freezer).Freeze(10) | ||
|
|
||
| frozen, err := db.Ancients() | ||
| //make sure there're frozen items | ||
| if err != nil || frozen == 0 { | ||
| t.Fatalf("Failed to import canonical chain start: %v", err) | ||
| } | ||
| if frozen < blockRemain { | ||
| t.Fatalf("block amount is not enough for pruning: %v", err) | ||
| } | ||
|
|
||
| oldOffSet := rawdb.ReadOffSetOfCurrentAncientFreezer(db) | ||
| // Get the actual start block number. | ||
| startBlockNumber := frozen - blockRemain + oldOffSet | ||
| // Initialize the slice to buffer the block data left. | ||
| blockList := make([]*types.Block, 0, blockPruneBackUpBlockNumber) | ||
| receiptsList := make([]types.Receipts, 0, blockPruneBackUpBlockNumber) | ||
| externTdList := make([]*big.Int, 0, blockPruneBackUpBlockNumber) | ||
| // All ancient data within the most recent 128 blocks write into memory buffer for future new ancient_back directory usage. | ||
| for blockNumber := startBlockNumber; blockNumber < frozen+oldOffSet; blockNumber++ { | ||
| blockHash := rawdb.ReadCanonicalHash(db, blockNumber) | ||
| block := rawdb.ReadBlock(db, blockHash, blockNumber) | ||
| blockList = append(blockList, block) | ||
| receipts := rawdb.ReadRawReceipts(db, blockHash, blockNumber) | ||
| receiptsList = append(receiptsList, receipts) | ||
| // Calculate the total difficulty of the block | ||
| td := rawdb.ReadTd(db, blockHash, blockNumber) | ||
| if td == nil { | ||
| t.Fatalf("Failed to ReadTd: %v", consensus.ErrUnknownAncestor) | ||
| } | ||
| externTd := new(big.Int).Add(block.Difficulty(), td) | ||
| externTdList = append(externTdList, externTd) | ||
| } | ||
|
|
||
| return db, blocks, blockList, receiptsList, externTdList, startBlockNumber, blockchain | ||
| } | ||
|
|
||
| func checkReceiptsRLP(have, want types.Receipts) error { | ||
| if len(have) != len(want) { | ||
| return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want)) | ||
| } | ||
| for i := 0; i < len(want); i++ { | ||
| rlpHave, err := rlp.EncodeToBytes(have[i]) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| rlpWant, err := rlp.EncodeToBytes(want[i]) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if !bytes.Equal(rlpHave, rlpWant) { | ||
| return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // startEthService creates a full node instance for testing. | ||
| func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block, chaindbPath string) (*node.Node, *eth.Ethereum) { | ||
| t.Helper() | ||
| n, err := node.New(&node.Config{DataDir: chaindbPath}) | ||
| if err != nil { | ||
| t.Fatal("can't create node:", err) | ||
| } | ||
|
|
||
| if err := n.Start(); err != nil { | ||
| t.Fatal("can't start node:", err) | ||
| } | ||
|
|
||
| return n, nil | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
unnecessry?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think it should be necessary? Because it will initialize the config, which will be used for GlobalFinding the value of ctx