Make db inspector for extending multiple ancient stores

core/blockchain.go

@@ -126,7 +126,7 @@ const (
  BlockChainVersion uint64 = 8
 )
 
-// CacheConfig contains the configuration values for the trie caching/pruning
+// CacheConfig contains the configuration values for the trie database
 // that's resident in a blockchain.
 type CacheConfig struct {
  TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory

core/rawdb/ancient_scheme.go

@@ -16,8 +16,6 @@
 
 package rawdb
 
-import "fmt"
-
 // The list of table names of chain freezer. (headers, hashes, bodies, difficulties)
 
 const (
@@ -54,36 +52,3 @@ var (
 
 // freezers the collections of all builtin freezers.
 var freezers = []string{chainFreezerName}
-
-// InspectFreezerTable dumps out the index of a specific freezer table. The passed
-// ancient indicates the path of root ancient directory where the chain freezer can
-// be opened. Start and end specifiy the range for dumping out indexes.
-// Note this function can only used for debugging purpose.
-func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
- var (
- path string
- tables map[string]bool
- )
-
- switch freezerName {
- case chainFreezerName:
- path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
- default:
- return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
- }
- noSnappy, exit := tables[tableName]
- if !exit {
- // If the tableName is not exit in the tables, return an error.
- var names []string
- for name := range tables {
- names = append(names, name)
- }
- return fmt.Errorf("unknown table name, supported ones: %v", names)
- }
- table, err := newFreezerTable(path, tableName, noSnappy)
- if err != nil {
- return err
- }
- table.dumpIndexStdout(start, end)
- return nil
-}

core/rawdb/ancient_utils.go

@@ -0,0 +1,125 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+)
+
+type tableSize struct {
+ name string
+ size common.StorageSize
+}
+
+// freezerInfo contains the basic information of the freezer.
+type freezerInfo struct {
+ name string // The identifier of freezer
+ head uint64 // The number of last stored item in the freezer
+ tail uint64 // The number of first stored item in the freezer
+ sizes []tableSize // The storage size per table
+}
+
+// count returns the number of stored items in the freezer.
+func (info *freezerInfo) count() uint64 {
+ return info.head - info.tail + 1
+}
+
+// size returns the storage size of the entire freezer.
+func (info *freezerInfo) size() common.StorageSize {
+ var total common.StorageSize
+ for _, table := range info.sizes {
+ total += table.size
+ }
+ return total
+}
+
+// inspectFreezers inspects all freezers registered in the system.
+func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
+ var (
+ infos []freezerInfo
+ )
+ for _, freezer := range freezers {
+ switch freezer {
+ case chainFreezerName: // We only support chain freezer for now.
+ // Chain ancient store is a bit special. It's always opened along
+ // with a key-value store, inspect the chain store directly.
+ info := freezerInfo{name: freezer}
+ // Retrieve storage size of every contained table.
+ for table := range chainFreezerNoSnappy {
+ size, err := db.AncientSize(table)
+ if err != nil {
+ return nil, err
+ }
+ info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
+ }
+ // Retrieve the number of last stored item
+ ancients, err := db.Ancients()
+ if err != nil {
+ return nil, err
+ }
+ info.head = ancients - 1
+
+ // Retrieve the number of first stored item
+ tail, err := db.Tail()
+ if err != nil {
+ return nil, err
+ }
+ info.tail = tail
+ infos = append(infos, info)
+ default:
+ return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+ }
+ }
+ return infos, nil
+}
+
+// InspectFreezerTable dumps out the index of a specific freezer table. The passed
+// ancient indicates the path of root ancient directory where the chain freezer can
+// be opened. Start and end specifiy the range for dumping out indexes.
+// Note this function can only used for debugging purpose.
+func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
+ var (
+ path string
+ tables map[string]bool
+ )
+
+ switch freezerName {
+ case chainFreezerName:
+ path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
+ default:
+ return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+ }
+ noSnappy, exit := tables[tableName]
+ if !exit {
+ // If the tableName is not exit in the tables, return an error.
+ var names []string
+ for name := range tables {
+ names = append(names, name)
+ }
+ return fmt.Errorf("unknown table name, supported ones: %v", names)
+ }
+ table, err := newFreezerTable(path, tableName, noSnappy)
+ if err != nil {
+ return err
+ }
+
+ table.dumpIndexStdout(start, end)
+ return nil
+}

core/rawdb/chain_freezer.go

@@ -73,6 +73,11 @@ func (f *chainFreezer) Close() error {
  return err
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (f *chainFreezer) Tail() (uint64, error) {
+ return 0, errNotSupported
+}
+
 // freeze is a background thread that periodically checks the blockchain for any
 // import progress and moves ancient data from the fast database into the freezer.
 //

core/rawdb/database.go

@@ -23,6 +23,7 @@ import (
  "os"
  "path"
  "path/filepath"
+ "strings"
  "time"
 
  "github.com/ethereum/go-ethereum/common"
@@ -87,6 +88,11 @@ type nofreezedb struct {
  ethdb.KeyValueStore
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (db *nofreezedb) Tail() (uint64, error) {
+ return 0, errNotSupported
+}
+
 // HasAncient returns an error as we don't have a backing chain freezer.
 func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) {
  return false, errNotSupported
@@ -445,13 +451,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
  cliqueSnaps stat
  consortiumSnaps stat
 
- // Ancient store statistics
- ancientHeadersSize common.StorageSize
- ancientBodiesSize common.StorageSize
- ancientReceiptsSize common.StorageSize
- ancientTdsSize common.StorageSize
- ancientHashesSize common.StorageSize
-
  // Les statistic
  chtTrieNodes stat
  bloomTrieNodes stat
@@ -540,20 +539,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
  logged = time.Now()
  }
  }
- // Inspect append-only file store then.
- ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
- for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
- if size, err := db.AncientSize(category); err == nil {
- *ancientSizes[i] += common.StorageSize(size)
- total += common.StorageSize(size)
- }
- }
- // Get number of ancient rows inside the freezer
- ancients := counter(0)
- if count, err := db.Ancients(); err == nil {
- ancients = counter(count)
- }
- // Display the database statistic.
+
+ // Display the database statistic of key-value store.
  stats := [][]string{
  {"Key-Value store", "Headers", headers.Size(), headers.Count()},
  {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
@@ -571,14 +558,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
  {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
  {"Key-Value store", "Consortium snapshots", consortiumSnaps.Size(), consortiumSnaps.Count()},
  {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
- {"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
- {"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
- {"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
- {"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
- {"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
  {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
  {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
  }
+ // Inspect all registered append-only file store then.
+ ancients, err := inspectFreezers(db)
+ if err != nil {
+ return err
+ }
+ for _, ancient := range ancients {
+ for _, table := range ancient.sizes {
+ stats = append(stats, []string{
+ fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
+ strings.Title(table.name),
+ table.size.String(),
+ fmt.Sprintf("%d", ancient.count()),
+ })
+ }
+ total += ancient.size()
+ }
  table := tablewriter.NewWriter(os.Stdout)
  table.SetHeader([]string{"Database", "Category", "Size", "Items"})
  table.SetFooter([]string{"", "Total", total.String(), " "})

core/rawdb/freezer.go

@@ -219,6 +219,12 @@ func (f *Freezer) AncientSize(kind string) (uint64, error) {
  return 0, errUnknownTable
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (f *Freezer) Tail() (uint64, error) {
+ // return f.tail.Load(), nil, in the next implementing, right now just keep it zero
+ return 0, nil
+}
+
 // ReadAncients runs the given read operation while ensuring that no writes take place
 // on the underlying freezer.
 func (f *Freezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {

core/rawdb/table.go

@@ -68,6 +68,12 @@ func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]by
  return t.db.AncientRange(kind, start, count, maxBytes)
 }
 
+// Tail is a noop passthrough that just forwards the request to the underlying
+// database.
+func (t *table) Tail() (uint64, error) {
+ return t.db.Tail()
+}
+
 // Ancients is a noop passthrough that just forwards the request to the underlying
 // database.
 func (t *table) Ancients() (uint64, error) {

core/state/state_object.go

@@ -392,9 +392,9 @@ func (s *stateObject) updateRoot(db Database) {
  s.data.Root = s.trie.Hash()
 }
 
-// CommitTrie the storage trie of the object to db.
-// This updates the trie root.
-func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
+// commitTrie submits the storage changes into the storage trie and re-computes
+// the root. Besides, all trie changes will be collected in a nodeset and returned.
+func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
  // If nothing changed, don't bother with hashing anything
  if s.updateTrie(db) == nil {
  return nil, nil

core/state/statedb.go

@@ -986,7 +986,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
  obj.dirtyCode = false
  }
  // Write any storage changes in the state object to its storage trie
- nodeSet, err := obj.CommitTrie(s.db)
+ nodeSet, err := obj.commitTrie(s.db)
  if err != nil {
  return common.Hash{}, err
  }
@@ -1000,6 +1000,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
  }
  }
  }
+ // If the contract is destructed, the storage is still left in the
+ // database as dangling data. Theoretically it's should be wiped from
+ // database as well, but in hash-based-scheme it's extremely hard to
+ // determine that if the trie nodes are also referenced by other storage,
+ // and in path-based-scheme some technical challenges are still unsolved.
+ // Although it won't affect the correctness but please fix it TODO(rjl493456442).
  if len(s.stateObjectsDirty) > 0 {
  s.stateObjectsDirty = make(map[common.Address]struct{})
  }

ethdb/database.go

@@ -87,6 +87,10 @@ type AncientReaderOp interface {
  // Ancients returns the ancient item numbers in the ancient store.
  Ancients() (uint64, error)
 
+ // Tail returns the number of first stored item in the ancient store.
+ // This number can also be interpreted as the total deleted items.
+ Tail() (uint64, error)
+
  // AncientSize returns the ancient size of the specified category.
  AncientSize(kind string) (uint64, error)
 }