From fe98df1c03101d23e8a52f0186f78a95f23b38fd Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 1 Dec 2021 14:24:38 +0100 Subject: [PATCH 01/10] cmd,core: add simple legacy receipt converter core/rawdb: use forEach in migrate core/rawdb: batch reads in forEach core/rawdb: make forEach anonymous fn cmd/geth: check for legacy receipts on node startup fix err msg Co-authored-by: rjl493456442 fix log Co-authored-by: rjl493456442 fix some review comments add warning to cmd drop isLegacy fn from migrateTable params add test for windows rename test replacing in windows case --- cmd/geth/config.go | 12 ++++- cmd/geth/dbcmd.go | 99 +++++++++++++++++++++++++++++++++++ core/rawdb/database.go | 6 +++ core/rawdb/freezer.go | 102 +++++++++++++++++++++++++++++++++++++ core/rawdb/freezer_test.go | 90 ++++++++++++++++++++++++++++++++ core/rawdb/table.go | 6 +++ core/types/legacy.go | 53 +++++++++++++++++++ ethdb/database.go | 5 ++ 8 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 core/types/legacy.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index ea4e65162997..d36a716784c2 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -161,7 +161,17 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) { cfg.Eth.OverrideTerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTerminalTotalDifficulty.Name)) } - backend, _ := utils.RegisterEthService(stack, &cfg.Eth) + backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + // Warn users to migrate if they have a legacy freezer format. + if eth != nil { + isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb()) + if err != nil { + utils.Fatalf("Failed to check db for legacy receipts: %v", err) + } + if isLegacy { + log.Warn("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") + } + } // Configure GraphQL if requested if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 4799a6388a9d..4732bd6f2afe 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" @@ -72,6 +73,7 @@ Remove blockchain and state databases`, dbImportCmd, dbExportCmd, dbMetadataCmd, + dbMigrateFreezerCmd, }, } dbInspectCmd = cli.Command{ @@ -251,6 +253,23 @@ WARNING: This is a low-level operation which may cause database corruption!`, }, Description: "Shows metadata about the chain status.", } + dbMigrateFreezerCmd = cli.Command{ + Action: utils.MigrateFlags(freezerMigrate), + Name: "freezer-migrate", + Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.SepoliaFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those. +WARNING: please back-up the receipt files in your ancients before running this command.`, + } ) func removeDB(ctx *cli.Context) error { @@ -750,3 +769,83 @@ func showMetaData(ctx *cli.Context) error { table.Render() return nil } + +func freezerMigrate(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false) + defer db.Close() + + // Check first block for legacy receipt format + numAncients, err := db.Ancients() + if err != nil { + return err + } + if numAncients < 1 { + log.Info("No receipts in freezer to migrate") + return nil + } + + isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db) + if err != nil { + return err + } + if !isFirstLegacy { + log.Info("No legacy receipts to migrate") + return nil + } + + log.Info("Starting migration", "ancients", numAncients, "firstLegacy", firstIdx) + start := time.Now() + if err := db.MigrateTable("receipts", types.ConvertLegacyStoredReceipts); err != nil { + return err + } + if err := db.Close(); err != nil { + return err + } + log.Info("Migration finished", "duration", time.Since(start)) + + return nil +} + +// dbHasLegacyReceipts checks freezer entries for legacy receipts. It stops at the first +// non-empty receipt and checks its format. The index of this first non-empty element is +// the second return parameter. +func dbHasLegacyReceipts(db ethdb.Database) (bool, uint64, error) { + // Check first block for legacy receipt format + numAncients, err := db.Ancients() + if err != nil { + return false, 0, err + } + if numAncients < 1 { + return false, 0, nil + } + var ( + legacy bool + firstIdx uint64 + blob []byte + emptyRLPList = []byte{192} + ) + // Find first block with non-empty receipt + for i := uint64(0); i < numAncients; i++ { + blob, err = db.Ancient("receipts", i) + if err != nil { + return false, 0, err + } + if len(blob) == 0 { + continue + } + if !bytes.Equal(blob, emptyRLPList) { + firstIdx = i + break + } + } + // Is first non-empty receipt legacy? + first, err := db.Ancient("receipts", firstIdx) + if err != nil { + return false, 0, err + } + legacy, err = types.IsLegacyStoredReceipts(first) + return legacy, firstIdx, err +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index be66828eeab8..6a38445c4fa6 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -145,6 +145,12 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReader) error) (e return fn(db) } +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { + return errNotSupported +} + // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index c7008e83f4d4..740149eaf6e5 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -19,6 +19,7 @@ package rawdb import ( "errors" "fmt" + "io/ioutil" "math" "os" "path/filepath" @@ -617,3 +618,104 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] return hashes, err } + +// convertLegacyFn takes a raw freezer entry in an older format and +// returns it in the new format. +type convertLegacyFn = func([]byte) ([]byte, error) + +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { + if f.readonly { + return errReadOnly + } + f.writeLock.Lock() + defer f.writeLock.Unlock() + + table, ok := f.tables[kind] + if !ok { + return errUnknownTable + } + // forEach iterates every entry in the table serially and in order, calling `fn` + // with the item as argument. If `fn` returns an error the iteration stops + // and that error will be returned. + forEach := func(t *freezerTable, fn func(uint64, []byte) error) error { + var ( + items = atomic.LoadUint64(&t.items) + batchSize = uint64(1024) + maxBytes = uint64(1024 * 1024) + ) + for i := uint64(0); i < items; { + if i+batchSize > items { + batchSize = items - i + } + data, err := t.RetrieveItems(i, batchSize, maxBytes) + if err != nil { + return err + } + for j, item := range data { + if err := fn(i+uint64(j), item); err != nil { + return err + } + } + i += uint64(len(data)) + } + return nil + } + ancientsPath := filepath.Dir(table.index.Name()) + // Set up new dir for the migrated table, the content of which + // we'll at the end move over to the ancients dir. + migrationPath := filepath.Join(ancientsPath, "migration") + newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind]) + if err != nil { + return err + } + var ( + batch = newTable.newBatch() + out []byte + start = time.Now() + logged = time.Now() + ) + // Iterate through entries and transform them + forEach(table, func(i uint64, blob []byte) error { + if i%10000 == 0 && time.Since(logged) > 16*time.Second { + log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + out, err = convert(blob) + if err != nil { + return err + } + if err := batch.AppendRaw(i, out); err != nil { + return err + } + return nil + }) + if err := batch.commit(); err != nil { + return err + } + // Release and delete old table files. Note this won't + // delete the index file. + table.releaseFilesAfter(0, true) + + if err := newTable.Close(); err != nil { + return err + } + files, err := ioutil.ReadDir(migrationPath) + if err != nil { + return err + } + // Move migrated files to ancients dir. + for _, f := range files { + // This will replace the old index file as a side-effect. + if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { + return err + } + } + // Delete by now empty dir. + if err := os.Remove(migrationPath); err != nil { + return err + } + + return nil +} diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index 74e3d660cb10..b3fd3059e750 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -24,6 +24,7 @@ import ( "math/big" "math/rand" "os" + "path" "sync" "testing" @@ -337,3 +338,92 @@ func checkAncientCount(t *testing.T, f *freezer, kind string, n uint64) { t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err) } } + +func TestRenameWindows(t *testing.T) { + var ( + fname = "file.bin" + fname2 = "file2.bin" + data = []byte{1, 2, 3, 4} + data2 = []byte{2, 3, 4, 5} + data3 = []byte{3, 5, 6, 7} + dataLen = 4 + ) + + // Create 2 temp dirs + dir1, err := os.MkdirTemp("", "rename-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(dir1) + dir2, err := os.MkdirTemp("", "rename-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(dir2) + + // Create file in dir1 and fill with data + f, err := os.Create(path.Join(dir1, fname)) + if err != nil { + t.Fatal(err) + } + f2, err := os.Create(path.Join(dir1, fname2)) + if err != nil { + t.Fatal(err) + } + f3, err := os.Create(path.Join(dir2, fname2)) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write(data); err != nil { + t.Fatal(err) + } + if _, err := f2.Write(data2); err != nil { + t.Fatal(err) + } + if _, err := f3.Write(data3); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + if err := f2.Close(); err != nil { + t.Fatal(err) + } + if err := f3.Close(); err != nil { + t.Fatal(err) + } + if err := os.Rename(f.Name(), path.Join(dir2, fname)); err != nil { + t.Fatal(err) + } + if err := os.Rename(f2.Name(), path.Join(dir2, fname2)); err != nil { + t.Fatal(err) + } + + // Check file contents + f, err = os.Open(path.Join(dir2, fname)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + buf := make([]byte, dataLen) + if _, err := f.Read(buf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, data) { + t.Errorf("unexpected file contents. Got %v\n", buf) + } + + f, err = os.Open(path.Join(dir2, fname2)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + if _, err := f.Read(buf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, data2) { + t.Errorf("unexpected file contents. Got %v\n", buf) + } +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index bb65e20a43a2..5eadf5f7c159 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -113,6 +113,12 @@ func (t *table) Sync() error { return t.db.Sync() } +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (t *table) MigrateTable(kind string, convert convertLegacyFn) error { + return t.db.MigrateTable(kind, convert) +} + // Put inserts the given value into the database at a prefixed version of the // provided key. func (t *table) Put(key []byte, value []byte) error { diff --git a/core/types/legacy.go b/core/types/legacy.go new file mode 100644 index 000000000000..9254381b1e11 --- /dev/null +++ b/core/types/legacy.go @@ -0,0 +1,53 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "errors" + + "github.com/ethereum/go-ethereum/rlp" +) + +// IsLegacyStoredReceipts tries to parse the RLP-encoded blob +// first as an array of v3 stored receipt, then v4 stored receipt and +// returns true if successful. +func IsLegacyStoredReceipts(raw []byte) (bool, error) { + var v3 []v3StoredReceiptRLP + if err := rlp.DecodeBytes(raw, &v3); err == nil { + return true, nil + } + var v4 []v4StoredReceiptRLP + if err := rlp.DecodeBytes(raw, &v4); err == nil { + return true, nil + } + var v5 []storedReceiptRLP + // Check to see valid fresh stored receipt + if err := rlp.DecodeBytes(raw, &v5); err == nil { + return false, nil + } + return false, errors.New("value is not a valid receipt encoding") +} + +// ConvertLegacyStoredReceipts takes the RLP encoding of an array of legacy +// stored receipts and returns a fresh RLP-encoded stored receipt. +func ConvertLegacyStoredReceipts(raw []byte) ([]byte, error) { + var receipts []ReceiptForStorage + if err := rlp.DecodeBytes(raw, &receipts); err != nil { + return nil, err + } + return rlp.EncodeToBytes(&receipts) +} diff --git a/ethdb/database.go b/ethdb/database.go index d749c255b9fd..b2e7c7228a0a 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -124,6 +124,11 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error + + // MigrateTable processes and migrates entries of a given table to a new format. + // The second argument is a function that takes a raw entry and returns it + // in the newest format. + MigrateTable(string, func([]byte) ([]byte, error)) error } // AncientWriteOp is given to the function argument of ModifyAncients. From 3d7ec7e971e606d665705bffa8a19958034fb897 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 17 Mar 2022 15:27:40 +0100 Subject: [PATCH 02/10] minor fix --- core/rawdb/freezer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 740149eaf6e5..1c2ff80daf1d 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -666,7 +666,7 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind]) + newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind], false) if err != nil { return err } From 52612e6a56e15560c0b1ce2498d15aa7869afff0 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 17 Mar 2022 17:20:10 +0100 Subject: [PATCH 03/10] sanity check for tail-deletion --- core/rawdb/freezer.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 1c2ff80daf1d..a7f45ad97606 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -662,6 +662,9 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { } return nil } + if table.itemOffset > 0 || table.itemHidden > 0 { + return fmt.Errorf("migration not supported for tail-deleted freezers") + } ancientsPath := filepath.Dir(table.index.Name()) // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. From 0b14470b2030bb9a786aa8191cc6f67d5415f272 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 17 Mar 2022 17:28:32 +0100 Subject: [PATCH 04/10] add log before moving files around --- core/rawdb/freezer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index a7f45ad97606..3895aad1cc2a 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -697,6 +697,7 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { if err := batch.commit(); err != nil { return err } + log.Info("Replacing old table files with migrated ones", "elapsed", common.PrettyDuration(time.Since(start))) // Release and delete old table files. Note this won't // delete the index file. table.releaseFilesAfter(0, true) From e08d82f3a0d5801d08a3a91d8be5809f5d85ee04 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 17 Mar 2022 17:58:17 +0100 Subject: [PATCH 05/10] speed-up hack for mainnet --- cmd/geth/config.go | 8 +++++++- cmd/geth/dbcmd.go | 35 ++++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index d36a716784c2..9d6934498747 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -164,7 +164,13 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Warn users to migrate if they have a legacy freezer format. if eth != nil { - isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb()) + firstIdx := uint64(0) + // Hack to speed up check for mainnet because we know + // the first non-empty block. + if ctx.GlobalIsSet(utils.MainnetFlag.Name) { + firstIdx = 46147 + } + isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx) if err != nil { utils.Fatalf("Failed to check db for legacy receipts: %v", err) } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 4732bd6f2afe..f7bebf56f97a 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -787,7 +787,7 @@ func freezerMigrate(ctx *cli.Context) error { return nil } - isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db) + isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db, 0) if err != nil { return err } @@ -812,7 +812,7 @@ func freezerMigrate(ctx *cli.Context) error { // dbHasLegacyReceipts checks freezer entries for legacy receipts. It stops at the first // non-empty receipt and checks its format. The index of this first non-empty element is // the second return parameter. -func dbHasLegacyReceipts(db ethdb.Database) (bool, uint64, error) { +func dbHasLegacyReceipts(db ethdb.Database, firstIdx uint64) (bool, uint64, error) { // Check first block for legacy receipt format numAncients, err := db.Ancients() if err != nil { @@ -821,24 +821,29 @@ func dbHasLegacyReceipts(db ethdb.Database) (bool, uint64, error) { if numAncients < 1 { return false, 0, nil } + if firstIdx >= numAncients { + return false, firstIdx, nil + } var ( legacy bool - firstIdx uint64 blob []byte emptyRLPList = []byte{192} ) - // Find first block with non-empty receipt - for i := uint64(0); i < numAncients; i++ { - blob, err = db.Ancient("receipts", i) - if err != nil { - return false, 0, err - } - if len(blob) == 0 { - continue - } - if !bytes.Equal(blob, emptyRLPList) { - firstIdx = i - break + // Find first block with non-empty receipt, only if + // the index is not already provided. + if firstIdx == 0 { + for i := uint64(0); i < numAncients; i++ { + blob, err = db.Ancient("receipts", i) + if err != nil { + return false, 0, err + } + if len(blob) == 0 { + continue + } + if !bytes.Equal(blob, emptyRLPList) { + firstIdx = i + break + } } } // Is first non-empty receipt legacy? From 8e4b4294c255e2c9185abfabfd352dbd6bc23db3 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 17 Mar 2022 18:23:35 +0100 Subject: [PATCH 06/10] fix mainnet check, use networkid instead --- cmd/geth/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 9d6934498747..44086e659517 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -167,7 +167,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { firstIdx := uint64(0) // Hack to speed up check for mainnet because we know // the first non-empty block. - if ctx.GlobalIsSet(utils.MainnetFlag.Name) { + if cfg.Eth.NetworkId == 1 { firstIdx = 46147 } isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx) From c503ec0b2bea2bf3da56ea3465ba6b3807c8f6ab Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 18 Mar 2022 15:30:37 +0100 Subject: [PATCH 07/10] check mainnet genesis --- cmd/geth/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 44086e659517..0050a51b8a26 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" @@ -167,7 +168,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { firstIdx := uint64(0) // Hack to speed up check for mainnet because we know // the first non-empty block. - if cfg.Eth.NetworkId == 1 { + ghash := rawdb.ReadCanonicalHash(eth.ChainDb(), 0) + if cfg.Eth.NetworkId == 1 && ghash == params.MainnetGenesisHash { firstIdx = 46147 } isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx) From 94e7e93391df5553835b9f2bbf96bd775031828c Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 23 Mar 2022 10:29:16 +0100 Subject: [PATCH 08/10] review fixes --- core/rawdb/freezer.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 3895aad1cc2a..6f5d1d604355 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -662,6 +662,8 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { } return nil } + // TODO(s1na): This is a sanity-check since as of now no process does tail-deletion. But the migration + // process assumes no deletion at tail and needs to be modified to account for that. if table.itemOffset > 0 || table.itemHidden > 0 { return fmt.Errorf("migration not supported for tail-deleted freezers") } @@ -680,7 +682,7 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { logged = time.Now() ) // Iterate through entries and transform them - forEach(table, func(i uint64, blob []byte) error { + if err := forEach(table, func(i uint64, blob []byte) error { if i%10000 == 0 && time.Since(logged) > 16*time.Second { log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() @@ -693,7 +695,9 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { return err } return nil - }) + }); err != nil { + return err + } if err := batch.commit(); err != nil { return err } From 561c180e7266f4c9963eddc57c55dd5c686a7605 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 23 Mar 2022 11:14:03 +0100 Subject: [PATCH 09/10] resume previous migration attempt --- core/rawdb/freezer.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 6f5d1d604355..5134c60b8f61 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -639,13 +639,13 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { // forEach iterates every entry in the table serially and in order, calling `fn` // with the item as argument. If `fn` returns an error the iteration stops // and that error will be returned. - forEach := func(t *freezerTable, fn func(uint64, []byte) error) error { + forEach := func(t *freezerTable, offset uint64, fn func(uint64, []byte) error) error { var ( items = atomic.LoadUint64(&t.items) batchSize = uint64(1024) maxBytes = uint64(1024 * 1024) ) - for i := uint64(0); i < items; { + for i := uint64(offset); i < items; { if i+batchSize > items { batchSize = items - i } @@ -680,9 +680,13 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { out []byte start = time.Now() logged = time.Now() + offset = newTable.items ) + if offset > 0 { + log.Info("found previous migration attempt", "migrated", offset) + } // Iterate through entries and transform them - if err := forEach(table, func(i uint64, blob []byte) error { + if err := forEach(table, offset, func(i uint64, blob []byte) error { if i%10000 == 0 && time.Since(logged) > 16*time.Second { log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() From 8a63c9957ba269db6792b1874ebe0352fc604b5f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 23 Mar 2022 15:17:36 +0100 Subject: [PATCH 10/10] core/rawdb: lint fix --- core/rawdb/freezer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 5134c60b8f61..8266933ee655 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -645,7 +645,7 @@ func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { batchSize = uint64(1024) maxBytes = uint64(1024 * 1024) ) - for i := uint64(offset); i < items; { + for i := offset; i < items; { if i+batchSize > items { batchSize = items - i }