From a360c1df285725543f1e98505eeecbe94e8462ce Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 22 Oct 2019 18:47:24 +0800 Subject: [PATCH 1/3] core/rawdb: check hash before return data from ancient db --- core/rawdb/accessors_chain.go | 132 +++++++++++++++++++++-------- core/rawdb/accessors_chain_test.go | 67 +++++++++++++++ 2 files changed, 163 insertions(+), 36 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index fab7ca56c3a5..2a5c72dedc65 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -173,18 +173,33 @@ func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) { // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerHeaderTable, number) - if len(data) == 0 { - data, _ = db.Get(headerKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerHeaderTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + // Then try to look up the data in leveldb. + data, _ = db.Get(headerKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerHeaderTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + return nil // Can't find the data anywhere. } // HasHeader verifies the existence of a block header corresponding to the hash. @@ -251,18 +266,33 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerBodiesTable, number) - if len(data) == 0 { - data, _ = db.Get(blockBodyKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerBodiesTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + // Then try to look up the data in leveldb. + data, _ = db.Get(blockBodyKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerBodiesTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + return nil // Can't find the data anywhere. } // WriteBodyRLP stores an RLP encoded block body into the database. @@ -315,18 +345,33 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding. func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerDifficultyTable, number) - if len(data) == 0 { - data, _ = db.Get(headerTDKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerDifficultyTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + // Then try to look up the data in leveldb. + data, _ = db.Get(headerTDKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerDifficultyTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + return nil // Can't find the data anywhere. } // ReadTd retrieves a block's total difficulty corresponding to the hash. @@ -375,18 +420,33 @@ func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { // ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // First try to look up the data in ancient database. Extra hash + // comparison is necessary since ancient database only maintains + // the canonical data. data, _ := db.Ancient(freezerReceiptTable, number) - if len(data) == 0 { - data, _ = db.Get(blockReceiptsKey(number, hash)) - // In the background freezer is moving data from leveldb to flatten files. - // So during the first check for ancient db, the data is not yet in there, - // but when we reach into leveldb, the data was already moved. That would - // result in a not found error. - if len(data) == 0 { - data, _ = db.Ancient(freezerReceiptTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data + } + } + // Then try to look up the data in leveldb. + data, _ = db.Get(blockReceiptsKey(number, hash)) + if len(data) > 0 { + return data + } + // In the background freezer is moving data from leveldb to flatten files. + // So during the first check for ancient db, the data is not yet in there, + // but when we reach into leveldb, the data was already moved. That would + // result in a not found error. + data, _ = db.Ancient(freezerReceiptTable, number) + if len(data) > 0 { + h, _ := db.Ancient(freezerHashTable, number) + if common.BytesToHash(h) == hash { + return data } } - return data + return nil // Can't find the data anywhere. } // ReadRawReceipts retrieves all the transaction receipts belonging to a block. diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 8c8affffd9e8..9b30c46a5040 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -22,6 +22,8 @@ import ( "fmt" "math/big" "testing" + "os" + "io/ioutil" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -358,3 +360,68 @@ func checkReceiptsRLP(have, want types.Receipts) error { } return nil } + +func TestAncientStorage(t *testing.T) { + // Freezer style fast import the chain. + frdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("failed to create temp freezer dir: %v", err) + } + defer os.Remove(frdir) + + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "") + if err != nil { + t.Fatalf("failed to create database with ancient backend") + } + // Create a test block + block := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(0), + Extra: []byte("test block"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyRootHash, + ReceiptHash: types.EmptyRootHash, + }) + // Ensure nothing non-existent will be read + hash, number := block.Hash(), block.NumberU64() + if blob := ReadHeaderRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent header returned") + } + if blob := ReadBodyRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent body returned") + } + if blob := ReadReceiptsRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent receipts returned") + } + if blob := ReadTdRLP(db, hash, number); len(blob) > 0 { + t.Fatalf("non existent td returned") + } + // Write and verify the header in the database + WriteAncientBlock(db, block, nil, big.NewInt(100)) + if blob := ReadHeaderRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no header returned") + } + if blob := ReadBodyRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no body returned") + } + if blob := ReadReceiptsRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no receipts returned") + } + if blob := ReadTdRLP(db, hash, number); len(blob) == 0 { + t.Fatalf("no td returned") + } + // Use a fake hash for data retrieval, nothing should be returned. + fakeHash := common.BytesToHash([]byte{0x01,0x02,0x03}) + if blob := ReadHeaderRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid header returned") + } + if blob := ReadBodyRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid body returned") + } + if blob := ReadReceiptsRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid receipts returned") + } + if blob := ReadTdRLP(db, fakeHash, number); len(blob) != 0 { + t.Fatalf("invalid td returned") + } +} + From 59a0610670e2fcf35a7b32ca9d488c77c1c277cf Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 30 Oct 2019 10:40:45 +0800 Subject: [PATCH 2/3] core/rawdb: fix lint --- core/rawdb/accessors_chain_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 9b30c46a5040..bb7dad5dffa1 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -20,10 +20,10 @@ import ( "bytes" "encoding/hex" "fmt" + "io/ioutil" "math/big" - "testing" "os" - "io/ioutil" + "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -410,7 +410,7 @@ func TestAncientStorage(t *testing.T) { t.Fatalf("no td returned") } // Use a fake hash for data retrieval, nothing should be returned. - fakeHash := common.BytesToHash([]byte{0x01,0x02,0x03}) + fakeHash := common.BytesToHash([]byte{0x01, 0x02, 0x03}) if blob := ReadHeaderRLP(db, fakeHash, number); len(blob) != 0 { t.Fatalf("invalid header returned") } @@ -424,4 +424,3 @@ func TestAncientStorage(t *testing.T) { t.Fatalf("invalid td returned") } } - From 18e74e9448ab7c372aa8a631fe3a8118e0ca0d16 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 19 Nov 2019 16:04:20 +0800 Subject: [PATCH 3/3] core/rawdb: calculate the hash in the fly --- core/rawdb/accessors_chain.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 2a5c72dedc65..4d064a33903f 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -177,11 +178,8 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu // comparison is necessary since ancient database only maintains // the canonical data. data, _ := db.Ancient(freezerHeaderTable, number) - if len(data) > 0 { - h, _ := db.Ancient(freezerHashTable, number) - if common.BytesToHash(h) == hash { - return data - } + if len(data) > 0 && crypto.Keccak256Hash(data) == hash { + return data } // Then try to look up the data in leveldb. data, _ = db.Get(headerKey(number, hash)) @@ -193,11 +191,8 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu // but when we reach into leveldb, the data was already moved. That would // result in a not found error. data, _ = db.Ancient(freezerHeaderTable, number) - if len(data) > 0 { - h, _ := db.Ancient(freezerHashTable, number) - if common.BytesToHash(h) == hash { - return data - } + if len(data) > 0 && crypto.Keccak256Hash(data) == hash { + return data } return nil // Can't find the data anywhere. }