chore: versa db test

cmd/geth/chaincmd.go

@@ -833,7 +833,7 @@ func dump(ctx *cli.Context) error {
  triedb := utils.MakeTrieDatabase(ctx, stack, db, true, true, false) // always enable preimage lookup
  defer triedb.Close()
 
- state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
+ state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb, false), nil)
  if err != nil {
  return err
  }

cmd/geth/dbcmd.go

@@ -18,6 +18,7 @@ package main
 
 import (
  "bytes"
+ "encoding/json"
  "errors"
  "fmt"
  "math"
@@ -34,6 +35,7 @@ import (
  "github.com/ethereum/go-ethereum/common/hexutil"
  "github.com/ethereum/go-ethereum/console/prompt"
  "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/core/state"
  "github.com/ethereum/go-ethereum/core/state/snapshot"
  "github.com/ethereum/go-ethereum/crypto"
  "github.com/ethereum/go-ethereum/ethdb"
@@ -91,6 +93,9 @@ Remove blockchain and state databases`,
  dbHbss2PbssCmd,
  dbTrieGetCmd,
  dbTrieDeleteCmd,
+ getVersionDBState,
+ getHashDBState,
+ diffDebugStateDB,
  },
  }
  dbInspectCmd = &cli.Command{
@@ -286,8 +291,141 @@ WARNING: This is a low-level operation which may cause database corruption!`,
  Description: `This commands will read current offset from kvdb, which is the current offset and starting BlockNumber
 of ancientStore, will also displays the reserved number of blocks in ancientStore `,
  }
+ getVersionDBState = &cli.Command{
+ Action: getDebugVersionState,
+ Name: "get-debug-version-state",
+ Flags: []cli.Flag{
+ utils.VersionStateDirFlag,
+ utils.BlockNumber,
+ },
+ }
+ getHashDBState = &cli.Command{
+ Action: getDebugHashState,
+ Name: "get-debug-hash-state",
+ Flags: []cli.Flag{
+ utils.HashStateDirFlag,
+ utils.BlockNumber,
+ },
+ }
+
+ diffDebugStateDB = &cli.Command{
+ Action: diffDebugState,
+ Name: "diff-debug-state",
+ Flags: []cli.Flag{
+ utils.VersionStateDirFlag,
+ utils.HashStateDirFlag,
+ utils.BlockNumber,
+ },
+ }
 )
 
+func diffDebugState(ctx *cli.Context) error {
+ if !ctx.IsSet(utils.VersionStateDirFlag.Name) {
+ return fmt.Errorf("please set `--versionstatedir` flag")
+ }
+ if !ctx.IsSet(utils.BlockNumber.Name) {
+ return fmt.Errorf("please set `--block` flag")
+ }
+ if !ctx.IsSet(utils.HashStateDirFlag.Name) {
+ return fmt.Errorf("please set `--hashstatedir` flag")
+ }
+ verDir := ctx.String(utils.VersionStateDirFlag.Name)
+ hasDir := ctx.String(utils.HashStateDirFlag.Name)
+ block := ctx.Int64(utils.BlockNumber.Name)
+
+ vdb, err := rawdb.Open(rawdb.OpenOptions{
+ ReadOnly: true,
+ Type: "pebble",
+ Directory: verDir,
+ })
+ if err != nil {
+ return err
+ }
+ verData, err := vdb.Get(state.DebugVersionStateKey(block))
+ if err != nil {
+ return err
+ }
+
+ verDebugState := &state.DebugVersionState{}
+ err = json.Unmarshal(verData, verDebugState)
+ if err != nil {
+ return nil
+ }
+
+ hdb, err := rawdb.Open(rawdb.OpenOptions{
+ ReadOnly: true,
+ Type: "pebble",
+ Directory: hasDir,
+ })
+ if err != nil {
+ return err
+ }
+ hashData, err := hdb.Get(state.DebugHashStateKey(block))
+ if err != nil {
+ return err
+ }
+ hasDebugState := &state.DebugHashState{}
+ err = json.Unmarshal(hashData, hasDebugState)
+ if err != nil {
+ return err
+ }
+
+ res := state.GenerateDebugStateDiff(verDebugState, hasDebugState)
+ fmt.Println(res)
+
+ return nil
+}
+
+func getDebugVersionState(ctx *cli.Context) error {
+ if !ctx.IsSet(utils.VersionStateDirFlag.Name) {
+ return fmt.Errorf("please set `--versionstatedir` flag")
+ }
+ if !ctx.IsSet(utils.BlockNumber.Name) {
+ return fmt.Errorf("please set `--block` flag")
+ }
+ dir := ctx.String(utils.VersionStateDirFlag.Name)
+ block := ctx.Int64(utils.BlockNumber.Name)
+ db, err := rawdb.Open(rawdb.OpenOptions{
+ ReadOnly: true,
+ Type: "pebble",
+ Directory: dir,
+ })
+ if err != nil {
+ return err
+ }
+ data, err := db.Get(state.DebugVersionStateKey(block))
+ if err != nil {
+ return err
+ }
+ fmt.Println(string(data))
+ return nil
+}
+
+func getDebugHashState(ctx *cli.Context) error {
+ if !ctx.IsSet(utils.HashStateDirFlag.Name) {
+ return fmt.Errorf("please set `--hashstatedir` flag")
+ }
+ if !ctx.IsSet(utils.BlockNumber.Name) {
+ return fmt.Errorf("please set `--block` flag")
+ }
+ dir := ctx.String(utils.HashStateDirFlag.Name)
+ block := ctx.Int64(utils.BlockNumber.Name)
+ db, err := rawdb.Open(rawdb.OpenOptions{
+ ReadOnly: true,
+ Type: "pebble",
+ Directory: dir,
+ })
+ if err != nil {
+ return err
+ }
+ data, err := db.Get(state.DebugHashStateKey(block))
+ if err != nil {
+ return err
+ }
+ fmt.Println(string(data))
+ return nil
+}
+
 func removeDB(ctx *cli.Context) error {
  stack, config := makeConfigNode(ctx)
 

cmd/geth/main.go

@@ -361,6 +361,8 @@ func geth(ctx *cli.Context) error {
  defer stack.Close()
 
  startNode(ctx, stack, backend, false)
+ // TODO:: debug code , will be deleted in the future
+ debug.StartPProf("127.0.0.1:7060", !ctx.IsSet("metrics.addr"))
  stack.Wait()
  return nil
 }

cmd/utils/cmd.go

@@ -343,7 +343,7 @@ func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block
  continue
  }
  // If we're above the chain head, state availability is a must
- if !chain.HasBlockAndState(block.Hash(), block.NumberU64()) {
+ if !chain.HasBlockAndState(block.Hash(), block.Number().Int64()) {
  return blocks[i:]
  }
  }

cmd/utils/flags.go

@@ -353,7 +353,7 @@ var (
  }
  StateSchemeFlag = &cli.StringFlag{
  Name: "state.scheme",
- Usage: "Scheme to use for storing ethereum state ('hash' or 'path')",
+ Usage: "Scheme to use for storing ethereum state ('hash', 'path', 'version')",
  Category: flags.StateCategory,
  }
  PathDBSyncFlag = &cli.BoolFlag{
@@ -1135,6 +1135,25 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
  Value: params.DefaultExtraReserveForBlobRequests,
  Category: flags.MiscCategory,
  }
+
+ BlockNumber = &cli.Int64Flag{
+ Name: "block",
+ Value: int64(0),
+ }
+
+ VersionStateDirFlag = &flags.DirectoryFlag{
+ Name: "versionstatedir",
+ Usage: "Data directory for the version databases and keystore",
+ Value: flags.DirectoryString(node.DefaultDataDir()),
+ Category: flags.EthCategory,
+ }
+
+ HashStateDirFlag = &flags.DirectoryFlag{
+ Name: "hashstatedir",
+ Usage: "Data directory for the version databases and keystore",
+ Value: flags.DirectoryString(node.DefaultDataDir()),
+ Category: flags.EthCategory,
+ }
 )
 
 var (
@@ -1953,11 +1972,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
  if ctx.IsSet(StateHistoryFlag.Name) {
  cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
  }
- scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
- if err != nil {
- Fatalf("%v", err)
+ if ctx.String(StateSchemeFlag.Name) != rawdb.VersionScheme {
+ scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
+ if err != nil {
+ Fatalf("%v", err)
+ }
+ cfg.StateScheme = scheme
+ } else {
+ // TODO:: compatible with cli line and configuration file, currently only supports cli.
+ cfg.StateScheme = rawdb.VersionScheme
  }
- cfg.StateScheme = scheme
+
  // Parse transaction history flag, if user is still using legacy config
  // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
  if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {

core/block_validator.go

@@ -25,6 +25,7 @@ import (
  "github.com/ethereum/go-ethereum/consensus"
  "github.com/ethereum/go-ethereum/core/state"
  "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/metrics"
  "github.com/ethereum/go-ethereum/params"
  "github.com/ethereum/go-ethereum/trie"
 )
@@ -40,6 +41,12 @@ func EnableRemoteVerifyManager(remoteValidator *remoteVerifyManager) BlockValida
  }
 }
 
+var (
+ validateBloomTimer = metrics.NewRegisteredTimer("validate/bloom/time", nil)
+ validateReceiptTimer = metrics.NewRegisteredTimer("validate/receipt/time", nil)
+ validateRootTimer = metrics.NewRegisteredTimer("validate/root/time", nil)
+)
+
 // BlockValidator is responsible for validating block headers, uncles and
 // processed state.
 //
@@ -96,7 +103,7 @@ func ValidateListsInBody(block *types.Block) error {
 // validated at this point.
 func (v *BlockValidator) ValidateBody(block *types.Block) error {
  // Check whether the block is already imported.
- if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
+ if v.bc.HasBlockAndState(block.Hash(), block.Number().Int64()) {
  return ErrKnownBlock
  }
  if v.bc.isCachedBadBlock(block) {
@@ -142,7 +149,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
  return nil
  },
  func() error {
- if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+ if !v.bc.HasBlockAndState(block.ParentHash(), block.Number().Int64()-1) {
  if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
  return consensus.ErrUnknownAncestor
  }
@@ -184,13 +191,20 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
  // For valid blocks this should always validate to true.
  validateFuns := []func() error{
  func() error {
+ defer func(start time.Time) {
+ validateBloomTimer.UpdateSince(start)
+ }(time.Now())
+
  rbloom := types.CreateBloom(receipts)
  if rbloom != header.Bloom {
  return fmt.Errorf("invalid bloom (remote: %x local: %x)", header.Bloom, rbloom)
  }
  return nil
  },
  func() error {
+ defer func(start time.Time) {
+ validateReceiptTimer.UpdateSince(start)
+ }(time.Now())
  receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil))
  if receiptSha != header.ReceiptHash {
  return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
@@ -209,6 +223,9 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
  })
  } else {
  validateFuns = append(validateFuns, func() error {
+ defer func(start time.Time) {
+ validateRootTimer.UpdateSince(start)
+ }(time.Now())
  if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
  return fmt.Errorf("invalid merkle root (remote: %x local: %x) dberr: %w", header.Root, root, statedb.Error())
  }