cmd/evm: unify staterunner and blockrunner more, add human-readable o…

…utput, stateless cross-checking option

cmd/evm/blockrunner.go

@@ -26,75 +26,83 @@ import (
 
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/tracing"
-	"github.com/ethereum/go-ethereum/eth/tracers/logger"
+	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
 
-var RunFlag = &cli.StringFlag{
-	Name:  "run",
-	Value: ".*",
-	Usage: "Run only those tests matching the regular expression.",
-}
-
 var blockTestCommand = &cli.Command{
 	Action:    blockTestCmd,
 	Name:      "blocktest",
 	Usage:     "Executes the given blockchain tests",
-	ArgsUsage: "<file>",
-	Flags:     []cli.Flag{RunFlag},
+	ArgsUsage: "<path>",
+	Flags: flags.Merge([]cli.Flag{
+		DumpFlag,
+		HumanReadableFlag,
+		RunFlag,
+		WitnessCrossCheckFlag,
+	}, traceFlags),
 }
 
 func blockTestCmd(ctx *cli.Context) error {
-	if len(ctx.Args().First()) == 0 {
-		return errors.New("path-to-test argument required")
+	path := ctx.Args().First()
+	if len(path) == 0 {
+		return errors.New("path argument required")
 	}
-
-	var tracer *tracing.Hooks
-	// Configure the EVM logger
-	if ctx.Bool(MachineFlag.Name) {
-		tracer = logger.NewJSONLogger(&logger.Config{
-			EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
-			DisableStack:     ctx.Bool(DisableStackFlag.Name),
-			DisableStorage:   ctx.Bool(DisableStorageFlag.Name),
-			EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name),
-		}, os.Stderr)
+	var (
+		collected = collectJSONFiles(path)
+		results   []testResult
+	)
+	for _, fname := range collected {
+		r, err := runBlockTest(ctx, fname)
+		if err != nil {
+			return err
+		}
+		results = append(results, r...)
 	}
-	// Load the test content from the input file
-	src, err := os.ReadFile(ctx.Args().First())
+	report(ctx, results)
+	return nil
+}
+
+func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) {
+	src, err := os.ReadFile(fname)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	var tests map[string]tests.BlockTest
+	var tests map[string]*tests.BlockTest
 	if err = json.Unmarshal(src, &tests); err != nil {
-		return err
+		return nil, err
 	}
 	re, err := regexp.Compile(ctx.String(RunFlag.Name))
 	if err != nil {
-		return fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err)
+		return nil, fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err)
 	}
+	tracer := tracerFromFlags(ctx)
 
-	// Run them in order
+	// Pull out keys to sort and ensure tests are run in order.
 	var keys []string
 	for key := range tests {
 		keys = append(keys, key)
 	}
 	sort.Strings(keys)
+
+	// Run all the tests.
+	var results []testResult
 	for _, name := range keys {
 		if !re.MatchString(name) {
 			continue
 		}
-		test := tests[name]
-		if err := test.Run(false, rawdb.HashScheme, false, tracer, func(res error, chain *core.BlockChain) {
+		result := &testResult{Name: name, Pass: true}
+		if err := tests[name].Run(false, rawdb.HashScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) {
 			if ctx.Bool(DumpFlag.Name) {
-				if state, _ := chain.State(); state != nil {
-					fmt.Println(string(state.Dump(nil)))
+				if s, _ := chain.State(); s != nil {
+					result.State = dump(s)
 				}
 			}
 		}); err != nil {
-			return fmt.Errorf("test %v: %w", name, err)
+			result.Pass, result.Error = false, err.Error()
 		}
+		results = append(results, *result)
 	}
-	return nil
+	return results, nil
 }

cmd/evm/eest.go

@@ -0,0 +1,49 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import "regexp"
+
+// testMetadata provides more granular access to the test information encoded
+// within its filename by the execution spec test (EEST).
+type testMetadata struct {
+	fork       string
+	module     string // which python module gnerated the test, e.g. eip7702
+	file       string // exact file the test came from, e.g. test_gas.py
+	function   string // func that created the test, e.g. test_valid_mcopy_operations
+	parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs
+}
+
+// parseTestMetadata reads a test name and parses out more specific information
+// about the test.
+func parseTestMetadata(s string) *testMetadata {
+	var (
+		pattern = `tests\/([^\/]+)\/([^\/]+)\/([^:]+)::([^[]+)\[fork_([^-\]]+)-[^-]+-(.+)\]`
+		re      = regexp.MustCompile(pattern)
+	)
+	match := re.FindStringSubmatch(s)
+	if len(match) == 0 {
+		return nil
+	}
+	return &testMetadata{
+		fork:       match[5],
+		module:     match[2],
+		file:       match[3],
+		function:   match[4],
+		parameters: match[6],
+	}
+}

cmd/evm/main.go

@@ -19,79 +19,124 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/tracing"
+	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/urfave/cli/v2"
 )
 
+// Some other nice-to-haves:
+// * accumulate traces into an object to bundle with test
+// * write tx identifier for trace before hand (blocktest only)
+// * combine blocktest and statetest runner logic using unified test interface
+
 const traceCategory = "TRACING"
 
 var (
-	// Debug flags.
+	// Test running flags.
+	RunFlag = &cli.StringFlag{
+		Name:  "run",
+		Value: ".*",
+		Usage: "Run only those tests matching the regular expression.",
+	}
+	WitnessCrossCheckFlag = &cli.BoolFlag{
+		Name:    "cross-check",
+		Aliases: []string{"xc"},
+		Usage:   "Cross-check stateful execution against stateless, verifying the witness generation.",
+	}
+
+	// Debugging flags.
 	DumpFlag = &cli.BoolFlag{
 		Name:  "dump",
 		Usage: "dumps the state after the run",
 	}
-
+	HumanReadableFlag = &cli.BoolFlag{
+		Name:  "human",
+		Usage: "\"Human-readable\" output",
+	}
 	StatDumpFlag = &cli.BoolFlag{
 		Name:  "statdump",
 		Usage: "displays stack and heap memory information",
 	}
 
 	// Tracing flags.
-	DebugFlag = &cli.BoolFlag{
-		Name:     "debug",
-		Usage:    "output full trace logs",
+	TraceFlag = &cli.BoolFlag{
+		Name:     "trace",
+		Usage:    "Enable tracing and output trace log.",
 		Category: traceCategory,
 	}
-	MachineFlag = &cli.BoolFlag{
-		Name:     "json",
-		Usage:    "output trace logs in machine readable format (json)",
+	TraceFormatFlag = &cli.StringFlag{
+		Name:     "trace.format",
+		Usage:    "Trace output format to use (struct|json)",
+		Value:    "struct",
 		Category: traceCategory,
 	}
-
-	DisableMemoryFlag = &cli.BoolFlag{
-		Name:     "nomemory",
+	TraceDisableMemoryFlag = &cli.BoolFlag{
+		Name:     "trace.nomemory",
+		Aliases:  []string{"nomemory"},
 		Value:    true,
 		Usage:    "disable memory output",
 		Category: traceCategory,
 	}
-	DisableStackFlag = &cli.BoolFlag{
-		Name:     "nostack",
+	TraceDisableStackFlag = &cli.BoolFlag{
+		Name:     "trace.nostack",
+		Aliases:  []string{"nostack"},
 		Usage:    "disable stack output",
 		Category: traceCategory,
 	}
-	DisableStorageFlag = &cli.BoolFlag{
-		Name:     "nostorage",
+	TraceDisableStorageFlag = &cli.BoolFlag{
+		Name:     "trace.nostorage",
+		Aliases:  []string{"nostorage"},
 		Usage:    "disable storage output",
 		Category: traceCategory,
 	}
-	DisableReturnDataFlag = &cli.BoolFlag{
-		Name:     "noreturndata",
+	TraceDisableReturnDataFlag = &cli.BoolFlag{
+		Name:     "trace.noreturndata",
+		Aliases:  []string{"noreturndata"},
 		Value:    true,
 		Usage:    "enable return data output",
 		Category: traceCategory,
 	}
+
+	// Deprecated flags.
+	DebugFlag = &cli.BoolFlag{
+		Name:     "debug",
+		Usage:    "output full trace logs (deprecated)",
+		Hidden:   true,
+		Category: traceCategory,
+	}
+	MachineFlag = &cli.BoolFlag{
+		Name:     "json",
+		Usage:    "output trace logs in machine readable format, json (deprecated)",
+		Hidden:   true,
+		Category: traceCategory,
+	}
 )
 
 // traceFlags contains flags that configure tracing output.
 var traceFlags = []cli.Flag{
+	TraceFlag,
+	TraceFormatFlag,
+	TraceDisableMemoryFlag,
+	TraceDisableStackFlag,
+	TraceDisableStorageFlag,
+	TraceDisableReturnDataFlag,
+
+	// deprecated
 	DebugFlag,
-	DumpFlag,
 	MachineFlag,
-	StatDumpFlag,
-	DisableMemoryFlag,
-	DisableStackFlag,
-	DisableStorageFlag,
-	DisableReturnDataFlag,
 }
 
 var app = flags.NewApp("the evm command line interface")
 
 func init() {
-	app.Flags = flags.Merge(traceFlags, debug.Flags)
+	app.Flags = flags.Merge(debug.Flags)
 	app.Commands = []*cli.Command{
 		runCommand,
 		blockTestCommand,
@@ -113,3 +158,46 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// tracerFromFlags parses the cli flags and returns the specified tracer.
+func tracerFromFlags(ctx *cli.Context) *tracing.Hooks {
+	config := &logger.Config{
+		EnableMemory:     !ctx.Bool(TraceDisableMemoryFlag.Name),
+		DisableStack:     ctx.Bool(TraceDisableStackFlag.Name),
+		DisableStorage:   ctx.Bool(TraceDisableStorageFlag.Name),
+		EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name),
+	}
+	switch {
+	case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "struct":
+		return logger.NewStructLogger(config).Hooks()
+	case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "json":
+		return logger.NewJSONLogger(config, os.Stderr)
+	case ctx.Bool(MachineFlag.Name):
+		return logger.NewJSONLogger(config, os.Stderr)
+	case ctx.Bool(DebugFlag.Name):
+		return logger.NewStructLogger(config).Hooks()
+	default:
+		return nil
+	}
+}
+
+// collectJSONFiles walks the given path and accumulates all files with json
+// extension.
+func collectJSONFiles(path string) []string {
+	var out []string
+	filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
+		if !info.IsDir() && filepath.Ext(info.Name()) == ".json" {
+			out = append(out, path)
+		}
+		return nil
+	})
+	return out
+}
+
+// dump returns a state dump for the most current trie.
+func dump(s *state.StateDB) *state.Dump {
+	root := s.IntermediateRoot(false)
+	cpy, _ := state.New(root, s.Database())
+	dump := cpy.RawDump(nil)
+	return &dump
+}