cmd/evm: unify staterunner and blockrunner more, add human-readable o…

…utput, stateless cross-checking option

cmd/evm/blockrunner.go

@@ -22,79 +22,87 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"slices"
 	"sort"
 
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/tracing"
-	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
 
-var RunFlag = &cli.StringFlag{
-	Name:  "run",
-	Value: ".*",
-	Usage: "Run only those tests matching the regular expression.",
-}
-
 var blockTestCommand = &cli.Command{
 	Action:    blockTestCmd,
 	Name:      "blocktest",
 	Usage:     "Executes the given blockchain tests",
-	ArgsUsage: "<file>",
-	Flags:     []cli.Flag{RunFlag},
+	ArgsUsage: "<path>",
+	Flags: slices.Concat([]cli.Flag{
+		DumpFlag,
+		HumanReadableFlag,
+		RunFlag,
+		WitnessCrossCheckFlag,
+	}, traceFlags),
 }
 
 func blockTestCmd(ctx *cli.Context) error {
-	if len(ctx.Args().First()) == 0 {
-		return errors.New("path-to-test argument required")
+	path := ctx.Args().First()
+	if len(path) == 0 {
+		return errors.New("path argument required")
 	}
-
-	var tracer *tracing.Hooks
-	// Configure the EVM logger
-	if ctx.Bool(MachineFlag.Name) {
-		tracer = logger.NewJSONLogger(&logger.Config{
-			EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
-			DisableStack:     ctx.Bool(DisableStackFlag.Name),
-			DisableStorage:   ctx.Bool(DisableStorageFlag.Name),
-			EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name),
-		}, os.Stderr)
+	var (
+		collected = collectJSONFiles(path)
+		results   []testResult
+	)
+	for _, fname := range collected {
+		r, err := runBlockTest(ctx, fname)
+		if err != nil {
+			return err
+		}
+		results = append(results, r...)
 	}
-	// Load the test content from the input file
-	src, err := os.ReadFile(ctx.Args().First())
+	report(ctx, results)
+	return nil
+}
+
+func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) {
+	src, err := os.ReadFile(fname)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	var tests map[string]tests.BlockTest
+	var tests map[string]*tests.BlockTest
 	if err = json.Unmarshal(src, &tests); err != nil {
-		return err
+		return nil, err
 	}
 	re, err := regexp.Compile(ctx.String(RunFlag.Name))
 	if err != nil {
-		return fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err)
+		return nil, fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err)
 	}
+	tracer := tracerFromFlags(ctx)
 
-	// Run them in order
+	// Pull out keys to sort and ensure tests are run in order.
 	var keys []string
 	for key := range tests {
 		keys = append(keys, key)
 	}
 	sort.Strings(keys)
+
+	// Run all the tests.
+	var results []testResult
 	for _, name := range keys {
 		if !re.MatchString(name) {
 			continue
 		}
-		test := tests[name]
-		if err := test.Run(false, rawdb.HashScheme, false, tracer, func(res error, chain *core.BlockChain) {
+		result := &testResult{Name: name, Pass: true}
+		if err := tests[name].Run(false, rawdb.HashScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) {
 			if ctx.Bool(DumpFlag.Name) {
-				if state, _ := chain.State(); state != nil {
-					fmt.Println(string(state.Dump(nil)))
+				if s, _ := chain.State(); s != nil {
+					result.State = dump(s)
 				}
 			}
 		}); err != nil {
-			return fmt.Errorf("test %v: %w", name, err)
+			result.Pass, result.Error = false, err.Error()
 		}
+		results = append(results, *result)
 	}
-	return nil
+	return results, nil
 }

cmd/evm/eest.go

@@ -0,0 +1,49 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import "regexp"
+
+// testMetadata provides more granular access to the test information encoded
+// within its filename by the execution spec test (EEST).
+type testMetadata struct {
+	fork       string
+	module     string // which python module gnerated the test, e.g. eip7702
+	file       string // exact file the test came from, e.g. test_gas.py
+	function   string // func that created the test, e.g. test_valid_mcopy_operations
+	parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs
+}
+
+// parseTestMetadata reads a test name and parses out more specific information
+// about the test.
+func parseTestMetadata(s string) *testMetadata {
+	var (
+		pattern = `tests\/([^\/]+)\/([^\/]+)\/([^:]+)::([^[]+)\[fork_([^-\]]+)-[^-]+-(.+)\]`
+		re      = regexp.MustCompile(pattern)
+	)
+	match := re.FindStringSubmatch(s)
+	if len(match) == 0 {
+		return nil
+	}
+	return &testMetadata{
+		fork:       match[5],
+		module:     match[2],
+		file:       match[3],
+		function:   match[4],
+		parameters: match[6],
+	}
+}

cmd/evm/eofparse.go

@@ -31,13 +31,41 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+var jt vm.JumpTable
+
+const initcode = "INITCODE"
+
 func init() {
 	jt = vm.NewPragueEOFInstructionSetForTesting()
 }
 
 var (
-	jt       vm.JumpTable
-	initcode = "INITCODE"
+	hexFlag = &cli.StringFlag{
+		Name:  "hex",
+		Usage: "Single container data parse and validation",
+	}
+	refTestFlag = &cli.StringFlag{
+		Name:  "test",
+		Usage: "Path to EOF validation reference test.",
+	}
+	eofParseCommand = &cli.Command{
+		Name:    "eofparse",
+		Aliases: []string{"eof"},
+		Usage:   "Parses hex eof container and returns validation errors (if any)",
+		Action:  eofParseAction,
+		Flags: []cli.Flag{
+			hexFlag,
+			refTestFlag,
+		},
+	}
+	eofDumpCommand = &cli.Command{
+		Name:   "eofdump",
+		Usage:  "Parses hex eof container and prints out human-readable representation of the container.",
+		Action: eofDumpAction,
+		Flags: []cli.Flag{
+			hexFlag,
+		},
+	}
 )
 
 func eofParseAction(ctx *cli.Context) error {