From 4853af79745f515566c266a8180cc8cf44eefc4e Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Thu, 27 Aug 2020 12:53:25 +0200 Subject: [PATCH 01/10] database: added counters --- core/rawdb/database.go | 93 ++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 25 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 316b5addf3c4..8dd036ff6397 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "strconv" "sync/atomic" "time" @@ -238,6 +239,10 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer return frdb, nil } +func formatCounter(counter uint64) string { + return strconv.FormatUint(counter, 10) +} + // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database) error { @@ -280,6 +285,26 @@ func InspectDatabase(db ethdb.Database) error { // Meta- and unaccounted data metadata common.StorageSize unaccounted common.StorageSize + + // Totals + headerSizeCount uint64 + bodySizeCount uint64 + receiptSizeCount uint64 + tdSizeCount uint64 + numHashPairingCount uint64 + hashNumPairingCount uint64 + trieSizeCount uint64 + codeSizeCount uint64 + txlookupSizeCount uint64 + accountSnapSizeCount uint64 + storageSnapSizeCount uint64 + preimageSizeCount uint64 + bloomBitsSizeCount uint64 + cliqueSnapsSizeCount uint64 + chtTrieNodesCount uint64 + bloomTrieNodesCount uint64 + metadataCount uint64 + unaccountedCount uint64 ) // Inspect key-value database first. for it.Next() { @@ -291,47 +316,65 @@ func InspectDatabase(db ethdb.Database) error { switch { case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): tdSize += size + tdSizeCount++ case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): numHashPairing += size + numHashPairingCount++ case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): headerSize += size + headerSizeCount++ case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairing += size + hashNumPairingCount++ case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): bodySize += size + bodySizeCount++ case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): receiptSize += size + receiptSizeCount++ case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): txlookupSize += size + txlookupSizeCount++ case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): accountSnapSize += size + accountSnapSizeCount++ case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): storageSnapSize += size + storageSnapSizeCount++ case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): preimageSize += size + preimageSizeCount++ case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): bloomBitsSize += size + bloomBitsSizeCount++ case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: cliqueSnapsSize += size + cliqueSnapsSizeCount++ case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: chtTrieNodes += size + chtTrieNodesCount++ case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: bloomTrieNodes += size + bloomTrieNodesCount++ case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength: codeSize += size + codeSizeCount++ case len(key) == common.HashLength: trieSize += size + trieSizeCount++ default: var accounted bool for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { if bytes.Equal(key, meta) { metadata += size + metadataCount++ accounted = true break } } if !accounted { unaccounted += size + unaccountedCount++ } } count += 1 @@ -350,37 +393,37 @@ func InspectDatabase(db ethdb.Database) error { } // Display the database statistic. stats := [][]string{ - {"Key-Value store", "Headers", headerSize.String()}, - {"Key-Value store", "Bodies", bodySize.String()}, - {"Key-Value store", "Receipts", receiptSize.String()}, - {"Key-Value store", "Difficulties", tdSize.String()}, - {"Key-Value store", "Block number->hash", numHashPairing.String()}, - {"Key-Value store", "Block hash->number", hashNumPairing.String()}, - {"Key-Value store", "Transaction index", txlookupSize.String()}, - {"Key-Value store", "Bloombit index", bloomBitsSize.String()}, - {"Key-Value store", "Contract codes", codeSize.String()}, - {"Key-Value store", "Trie nodes", trieSize.String()}, - {"Key-Value store", "Trie preimages", preimageSize.String()}, - {"Key-Value store", "Account snapshot", accountSnapSize.String()}, - {"Key-Value store", "Storage snapshot", storageSnapSize.String()}, - {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, - {"Key-Value store", "Singleton metadata", metadata.String()}, - {"Ancient store", "Headers", ancientHeaders.String()}, - {"Ancient store", "Bodies", ancientBodies.String()}, - {"Ancient store", "Receipts", ancientReceipts.String()}, - {"Ancient store", "Difficulties", ancientTds.String()}, - {"Ancient store", "Block number->hash", ancientHashes.String()}, - {"Light client", "CHT trie nodes", chtTrieNodes.String()}, - {"Light client", "Bloom trie nodes", bloomTrieNodes.String()}, + {"Key-Value store", "Headers", headerSize.String(), formatCounter(headerSizeCount)}, + {"Key-Value store", "Bodies", bodySize.String(), formatCounter(bodySizeCount)}, + {"Key-Value store", "Receipts", receiptSize.String(), formatCounter(receiptSizeCount)}, + {"Key-Value store", "Difficulties", tdSize.String(), formatCounter(tdSizeCount)}, + {"Key-Value store", "Block number->hash", numHashPairing.String(), formatCounter(numHashPairingCount)}, + {"Key-Value store", "Block hash->number", hashNumPairing.String(), formatCounter(hashNumPairingCount)}, + {"Key-Value store", "Transaction index", txlookupSize.String(), formatCounter(txlookupSizeCount)}, + {"Key-Value store", "Bloombit index", bloomBitsSize.String(), formatCounter(bloomBitsSizeCount)}, + {"Key-Value store", "Contract codes", codeSize.String(), formatCounter(codeSizeCount)}, + {"Key-Value store", "Trie nodes", trieSize.String(), formatCounter(trieSizeCount)}, + {"Key-Value store", "Trie preimages", preimageSize.String(), formatCounter(preimageSizeCount)}, + {"Key-Value store", "Account snapshot", accountSnapSize.String(), formatCounter(accountSnapSizeCount)}, + {"Key-Value store", "Storage snapshot", storageSnapSize.String(), formatCounter(storageSnapSizeCount)}, + {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String(), formatCounter(cliqueSnapsSizeCount)}, + {"Key-Value store", "Singleton metadata", metadata.String(), formatCounter(metadataCount)}, + {"Ancient store", "Headers", ancientHeaders.String(), "NA"}, + {"Ancient store", "Bodies", ancientBodies.String(), "NA"}, + {"Ancient store", "Receipts", ancientReceipts.String(), "NA"}, + {"Ancient store", "Difficulties", ancientTds.String(), "NA"}, + {"Ancient store", "Block number->hash", ancientHashes.String(), "NA"}, + {"Light client", "CHT trie nodes", chtTrieNodes.String(), formatCounter(chtTrieNodesCount)}, + {"Light client", "Bloom trie nodes", bloomTrieNodes.String(), formatCounter(bloomTrieNodesCount)}, } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Database", "Category", "Size"}) - table.SetFooter([]string{"", "Total", total.String()}) + table.SetHeader([]string{"Database", "Category", "Size", "Count"}) + table.SetFooter([]string{"", "Total", total.String(), ""}) table.AppendBulk(stats) table.Render() if unaccounted > 0 { - log.Error("Database contains unaccounted data", "size", unaccounted) + log.Error("Database contains unaccounted data", "size", unaccounted, "count", unaccountedCount) } return nil } From 30a4edf5a7c3b6d1d70c0329b6a40ffb96d5d241 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 28 Aug 2020 03:37:51 +0200 Subject: [PATCH 02/10] Improved stats for ancient db --- core/rawdb/database.go | 247 ++++++++++++++++++++++------------------- 1 file changed, 133 insertions(+), 114 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 8dd036ff6397..70fab6aab0be 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,15 +21,16 @@ import ( "errors" "fmt" "os" - "strconv" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" "github.com/olekukonko/tablewriter" ) @@ -239,8 +240,41 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer return frdb, nil } -func formatCounter(counter uint64) string { - return strconv.FormatUint(counter, 10) +type counter uint64 + +func (c counter) String() string { + return fmt.Sprintf("%d", c) +} + +// stat stores sizes and count for a parameter +type stat struct { + size common.StorageSize + count counter +} + +// Add size to the stat and increase the counter by 1 +func (s *stat) Add(size common.StorageSize) { + s.size += size + s.count++ +} + +func (s *stat) Size() string { + return s.size.String() +} + +func (s *stat) Count() string { + return s.count.String() +} + +// countReceiptsRLP counts how many receipts are stored in a RLP raw value +func countReceiptsRLP(data rlp.RawValue) counter { + // Convert the receipts from their storage form to their internal representation + storageReceipts := []*types.ReceiptForStorage{} + if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { + log.Error("Invalid receipt array RLP") + return counter(0) + } + return counter(len(storageReceipts)) } // InspectDatabase traverses the entire database and checks the size @@ -255,56 +289,38 @@ func InspectDatabase(db ethdb.Database) error { logged = time.Now() // Key-value store statistics - total common.StorageSize - headerSize common.StorageSize - bodySize common.StorageSize - receiptSize common.StorageSize - tdSize common.StorageSize - numHashPairing common.StorageSize - hashNumPairing common.StorageSize - trieSize common.StorageSize - codeSize common.StorageSize - txlookupSize common.StorageSize - accountSnapSize common.StorageSize - storageSnapSize common.StorageSize - preimageSize common.StorageSize - bloomBitsSize common.StorageSize - cliqueSnapsSize common.StorageSize + headers stat + bodies stat + receipts stat + tds stat + numHashPairings stat + hashNumPairings stat + tries stat + codes stat + txLookups stat + accountSnaps stat + storageSnaps stat + preimages stat + bloomBits stat + cliqueSnaps stat // Ancient store statistics - ancientHeaders common.StorageSize - ancientBodies common.StorageSize - ancientReceipts common.StorageSize - ancientHashes common.StorageSize - ancientTds common.StorageSize + ancientHeadersSize common.StorageSize + ancientBodiesSize common.StorageSize + ancientReceiptsSize common.StorageSize + ancientTdsSize common.StorageSize + ancientHashesSize common.StorageSize // Les statistic - chtTrieNodes common.StorageSize - bloomTrieNodes common.StorageSize + chtTrieNodes stat + bloomTrieNodes stat // Meta- and unaccounted data - metadata common.StorageSize - unaccounted common.StorageSize + metadata stat + unaccounted stat // Totals - headerSizeCount uint64 - bodySizeCount uint64 - receiptSizeCount uint64 - tdSizeCount uint64 - numHashPairingCount uint64 - hashNumPairingCount uint64 - trieSizeCount uint64 - codeSizeCount uint64 - txlookupSizeCount uint64 - accountSnapSizeCount uint64 - storageSnapSizeCount uint64 - preimageSizeCount uint64 - bloomBitsSizeCount uint64 - cliqueSnapsSizeCount uint64 - chtTrieNodesCount uint64 - bloomTrieNodesCount uint64 - metadataCount uint64 - unaccountedCount uint64 + total common.StorageSize ) // Inspect key-value database first. for it.Next() { @@ -314,67 +330,49 @@ func InspectDatabase(db ethdb.Database) error { ) total += size switch { - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): - tdSize += size - tdSizeCount++ - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): - numHashPairing += size - numHashPairingCount++ case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): - headerSize += size - headerSizeCount++ - case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): - hashNumPairing += size - hashNumPairingCount++ + headers.Add(size) case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): - bodySize += size - bodySizeCount++ + bodies.Add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): - receiptSize += size - receiptSizeCount++ + receipts.size += size + receipts.count += countReceiptsRLP(it.Value()) + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): + tds.Add(size) + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): + numHashPairings.Add(size) + case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): + hashNumPairings.Add(size) + case len(key) == common.HashLength: + tries.Add(size) + case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength: + codes.Add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): - txlookupSize += size - txlookupSizeCount++ + txLookups.Add(size) case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): - accountSnapSize += size - accountSnapSizeCount++ + accountSnaps.Add(size) case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): - storageSnapSize += size - storageSnapSizeCount++ + storageSnaps.Add(size) case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): - preimageSize += size - preimageSizeCount++ + preimages.Add(size) case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): - bloomBitsSize += size - bloomBitsSizeCount++ + bloomBits.Add(size) case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: - cliqueSnapsSize += size - cliqueSnapsSizeCount++ + cliqueSnaps.Add(size) case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: - chtTrieNodes += size - chtTrieNodesCount++ + chtTrieNodes.Add(size) case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: - bloomTrieNodes += size - bloomTrieNodesCount++ - case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength: - codeSize += size - codeSizeCount++ - case len(key) == common.HashLength: - trieSize += size - trieSizeCount++ - default: + bloomTrieNodes.Add(size) var accounted bool for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { if bytes.Equal(key, meta) { - metadata += size - metadataCount++ + metadata.Add(size) accounted = true break } } if !accounted { - unaccounted += size - unaccountedCount++ + unaccounted.Add(size) } } count += 1 @@ -384,46 +382,67 @@ func InspectDatabase(db ethdb.Database) error { } } // Inspect append-only file store then. - ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds} + ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { if size, err := db.AncientSize(category); err == nil { - *ancients[i] += common.StorageSize(size) + *ancientSizes[i] += common.StorageSize(size) total += common.StorageSize(size) } } + // Get number of ancient rows inside the freezer + ancients := counter(0) + if count, err := db.Ancients(); err == nil { + ancients = counter(count) + } + // Count receipts in ancient db + ancientReceipts := counter(0) + for blockNumber := uint64(0); blockNumber <= uint64(ancients-1); blockNumber++ { + data, err := db.Ancient(freezerReceiptTable, blockNumber) + if err != nil { + log.Error("Error reading ancient receipts from block", "number", blockNumber, "err", err) + } else { + ancientReceipts += countReceiptsRLP(data) + } + if blockNumber%1000 == 0 && time.Since(logged) > 8*time.Second { + log.Info("Inspecting ancient database", "number", blockNumber, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } // Display the database statistic. stats := [][]string{ - {"Key-Value store", "Headers", headerSize.String(), formatCounter(headerSizeCount)}, - {"Key-Value store", "Bodies", bodySize.String(), formatCounter(bodySizeCount)}, - {"Key-Value store", "Receipts", receiptSize.String(), formatCounter(receiptSizeCount)}, - {"Key-Value store", "Difficulties", tdSize.String(), formatCounter(tdSizeCount)}, - {"Key-Value store", "Block number->hash", numHashPairing.String(), formatCounter(numHashPairingCount)}, - {"Key-Value store", "Block hash->number", hashNumPairing.String(), formatCounter(hashNumPairingCount)}, - {"Key-Value store", "Transaction index", txlookupSize.String(), formatCounter(txlookupSizeCount)}, - {"Key-Value store", "Bloombit index", bloomBitsSize.String(), formatCounter(bloomBitsSizeCount)}, - {"Key-Value store", "Contract codes", codeSize.String(), formatCounter(codeSizeCount)}, - {"Key-Value store", "Trie nodes", trieSize.String(), formatCounter(trieSizeCount)}, - {"Key-Value store", "Trie preimages", preimageSize.String(), formatCounter(preimageSizeCount)}, - {"Key-Value store", "Account snapshot", accountSnapSize.String(), formatCounter(accountSnapSizeCount)}, - {"Key-Value store", "Storage snapshot", storageSnapSize.String(), formatCounter(storageSnapSizeCount)}, - {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String(), formatCounter(cliqueSnapsSizeCount)}, - {"Key-Value store", "Singleton metadata", metadata.String(), formatCounter(metadataCount)}, - {"Ancient store", "Headers", ancientHeaders.String(), "NA"}, - {"Ancient store", "Bodies", ancientBodies.String(), "NA"}, - {"Ancient store", "Receipts", ancientReceipts.String(), "NA"}, - {"Ancient store", "Difficulties", ancientTds.String(), "NA"}, - {"Ancient store", "Block number->hash", ancientHashes.String(), "NA"}, - {"Light client", "CHT trie nodes", chtTrieNodes.String(), formatCounter(chtTrieNodesCount)}, - {"Light client", "Bloom trie nodes", bloomTrieNodes.String(), formatCounter(bloomTrieNodesCount)}, + {"Key-Value store", "Headers", headers.Size(), headers.Count()}, + {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, + {"Key-Value store", "Receipts", receipts.Size(), receipts.Count()}, + {"Key-Value store", "Difficulties", tds.Size(), tds.Count()}, + {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, + {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, + {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, + {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, + {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, + {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, + {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, + {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, + {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, + {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, + {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, + {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()}, + {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()}, + {"Ancient store", "Receipts", ancientReceiptsSize.String(), ancientReceipts.String()}, + {"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()}, + {"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()}, + {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, + {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, } table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Count"}) - table.SetFooter([]string{"", "Total", total.String(), ""}) + table.SetFooter([]string{"", "Total", total.String(), " "}) table.AppendBulk(stats) table.Render() - if unaccounted > 0 { - log.Error("Database contains unaccounted data", "size", unaccounted, "count", unaccountedCount) + if unaccounted.size > 0 { + log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) } + + log.Info("Inspection completed", "elapsed", common.PrettyDuration(time.Since(start))) return nil } From 2551b2acd1ea86c659464526d8b1ef632624ff72 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 28 Aug 2020 04:26:38 +0200 Subject: [PATCH 03/10] Small improvement --- core/rawdb/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 70fab6aab0be..6801a63ba1df 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -396,7 +396,7 @@ func InspectDatabase(db ethdb.Database) error { } // Count receipts in ancient db ancientReceipts := counter(0) - for blockNumber := uint64(0); blockNumber <= uint64(ancients-1); blockNumber++ { + for blockNumber := uint64(0); blockNumber < uint64(ancients); blockNumber++ { data, err := db.Ancient(freezerReceiptTable, blockNumber) if err != nil { log.Error("Error reading ancient receipts from block", "number", blockNumber, "err", err) From 88c5de0770d1bdcf19d03e96a4f37300e468ebc5 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 28 Aug 2020 05:45:19 +0200 Subject: [PATCH 04/10] Better message and added percentage while counting receipts --- core/rawdb/database.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 6801a63ba1df..d8d51755dbb7 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -246,6 +246,10 @@ func (c counter) String() string { return fmt.Sprintf("%d", c) } +func (c counter) Percentage(current uint64) string { + return fmt.Sprintf("%d", current*100/uint64(c)) +} + // stat stores sizes and count for a parameter type stat struct { size common.StorageSize @@ -404,7 +408,7 @@ func InspectDatabase(db ethdb.Database) error { ancientReceipts += countReceiptsRLP(data) } if blockNumber%1000 == 0 && time.Since(logged) > 8*time.Second { - log.Info("Inspecting ancient database", "number", blockNumber, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Counting ancient database receipts", "number", blockNumber, "percentage", ancients.Percentage(blockNumber), "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } } From 363aa0ba5d41ceecb0bb2213a9675befce5b2237 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 28 Aug 2020 06:05:39 +0200 Subject: [PATCH 05/10] Fast counting for receipts --- core/rawdb/database.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index d8d51755dbb7..59da5e486ab8 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -25,7 +25,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" @@ -272,13 +271,16 @@ func (s *stat) Count() string { // countReceiptsRLP counts how many receipts are stored in a RLP raw value func countReceiptsRLP(data rlp.RawValue) counter { - // Convert the receipts from their storage form to their internal representation - storageReceipts := []*types.ReceiptForStorage{} - if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { - log.Error("Invalid receipt array RLP") + it, err := rlp.NewListIterator(data) + if err != nil { + log.Warn("Receipt iteration error", "error", err) return counter(0) } - return counter(len(storageReceipts)) + count := counter(0) + for it.Next() { + count++ + } + return count } // InspectDatabase traverses the entire database and checks the size From 3ea8a876f1aa8105e82215349805cc2f67619844 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 28 Aug 2020 11:42:59 +0200 Subject: [PATCH 06/10] added info message --- core/rawdb/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 59da5e486ab8..629e4ef72373 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -410,10 +410,11 @@ func InspectDatabase(db ethdb.Database) error { ancientReceipts += countReceiptsRLP(data) } if blockNumber%1000 == 0 && time.Since(logged) > 8*time.Second { - log.Info("Counting ancient database receipts", "number", blockNumber, "percentage", ancients.Percentage(blockNumber), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Counting ancient database receipts", "blocknumber", blockNumber, "percentage", ancients.Percentage(blockNumber), "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } } + log.Info("Counting ancient database receipts", "blocknumber", uint64(ancients), "percentage", "100", "elapsed", common.PrettyDuration(time.Since(start))) // Display the database statistic. stats := [][]string{ {"Key-Value store", "Headers", headers.Size(), headers.Count()}, @@ -449,6 +450,5 @@ func InspectDatabase(db ethdb.Database) error { log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) } - log.Info("Inspection completed", "elapsed", common.PrettyDuration(time.Since(start))) return nil } From 91c739fef8c4f87a32f386d912cf6640d66833d1 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Fri, 28 Aug 2020 13:00:59 +0200 Subject: [PATCH 07/10] Show both receips itemscount from ancient db and counted receipts --- core/rawdb/database.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 629e4ef72373..f587fdebe3c1 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -434,14 +434,15 @@ func InspectDatabase(db ethdb.Database) error { {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()}, {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()}, - {"Ancient store", "Receipts", ancientReceiptsSize.String(), ancientReceipts.String()}, + {"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()}, + {"Ancient store", "└ counted receipts", "--", ancientReceipts.String()}, {"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()}, {"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()}, {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Database", "Category", "Size", "Count"}) + table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", total.String(), " "}) table.AppendBulk(stats) table.Render() From de80906419fb05d384534c2dda831bd35e12f570 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Sun, 30 Aug 2020 06:04:42 +0200 Subject: [PATCH 08/10] Fixed default case --- core/rawdb/database.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index f587fdebe3c1..0a607e4b6c8d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -369,6 +369,7 @@ func InspectDatabase(db ethdb.Database) error { chtTrieNodes.Add(size) case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: bloomTrieNodes.Add(size) + default: var accounted bool for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { if bytes.Equal(key, meta) { @@ -381,7 +382,7 @@ func InspectDatabase(db ethdb.Database) error { unaccounted.Add(size) } } - count += 1 + count++ if count%1000 == 0 && time.Since(logged) > 8*time.Second { log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() From 58aa971b90f3af042415c5f223a8bf9d07189121 Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Tue, 15 Sep 2020 00:31:38 +0200 Subject: [PATCH 09/10] Removed counter for receipts in ancient store --- core/rawdb/database.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 0a607e4b6c8d..c4f49536b6d4 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -401,21 +401,6 @@ func InspectDatabase(db ethdb.Database) error { if count, err := db.Ancients(); err == nil { ancients = counter(count) } - // Count receipts in ancient db - ancientReceipts := counter(0) - for blockNumber := uint64(0); blockNumber < uint64(ancients); blockNumber++ { - data, err := db.Ancient(freezerReceiptTable, blockNumber) - if err != nil { - log.Error("Error reading ancient receipts from block", "number", blockNumber, "err", err) - } else { - ancientReceipts += countReceiptsRLP(data) - } - if blockNumber%1000 == 0 && time.Since(logged) > 8*time.Second { - log.Info("Counting ancient database receipts", "blocknumber", blockNumber, "percentage", ancients.Percentage(blockNumber), "elapsed", common.PrettyDuration(time.Since(start))) - logged = time.Now() - } - } - log.Info("Counting ancient database receipts", "blocknumber", uint64(ancients), "percentage", "100", "elapsed", common.PrettyDuration(time.Since(start))) // Display the database statistic. stats := [][]string{ {"Key-Value store", "Headers", headers.Size(), headers.Count()}, @@ -436,7 +421,6 @@ func InspectDatabase(db ethdb.Database) error { {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()}, {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()}, {"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()}, - {"Ancient store", "└ counted receipts", "--", ancientReceipts.String()}, {"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()}, {"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()}, {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, From e1393ef967f116fc1fd34f9217f3810520a1a80a Mon Sep 17 00:00:00 2001 From: Giuseppe Bertone Date: Tue, 15 Sep 2020 14:12:00 +0200 Subject: [PATCH 10/10] Removed counting of receipts present in leveldb --- core/rawdb/database.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c4f49536b6d4..b1ac3e95878f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" "github.com/olekukonko/tablewriter" ) @@ -269,20 +268,6 @@ func (s *stat) Count() string { return s.count.String() } -// countReceiptsRLP counts how many receipts are stored in a RLP raw value -func countReceiptsRLP(data rlp.RawValue) counter { - it, err := rlp.NewListIterator(data) - if err != nil { - log.Warn("Receipt iteration error", "error", err) - return counter(0) - } - count := counter(0) - for it.Next() { - count++ - } - return count -} - // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database) error { @@ -341,8 +326,7 @@ func InspectDatabase(db ethdb.Database) error { case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): bodies.Add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): - receipts.size += size - receipts.count += countReceiptsRLP(it.Value()) + receipts.Add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): tds.Add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): @@ -405,7 +389,7 @@ func InspectDatabase(db ethdb.Database) error { stats := [][]string{ {"Key-Value store", "Headers", headers.Size(), headers.Count()}, {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, - {"Key-Value store", "Receipts", receipts.Size(), receipts.Count()}, + {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, {"Key-Value store", "Difficulties", tds.Size(), tds.Count()}, {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},