feat: firehose live tracer with hooks tracing

core/state/intra_block_state.go

@@ -566,6 +566,10 @@ func (sdb *IntraBlockState) createObject(addr libcommon.Address, previous *state
 		sdb.journal.append(resetObjectChange{account: &addr, prev: previous})
 	}
 
+	if sdb.logger != nil && sdb.logger.OnNewAccount != nil {
+		sdb.logger.OnNewAccount(addr, previous != nil)
+	}
+
 	newobj.newlyCreated = true
 	sdb.setStateObject(addr, newobj)
 	return newobj

core/tracing/hooks.go

@@ -181,6 +181,13 @@ type Hooks struct {
 	OnCodeChange    CodeChangeHook
 	OnStorageChange StorageChangeHook
 	OnLog           LogHook
+
+	// Firehose backward compatibility
+	// This hook exist because some current Firehose supported chains requires it
+	// but this field is going to be deprecated and newer chains will not produced
+	// those events anymore. The hook is registered conditionally based on the
+	// tracer configuration.
+	OnNewAccount func(address libcommon.Address, previousExisted bool)
 }
 
 // BalanceChangeReason is used to indicate the reason for a balance change, useful

core/vm/runtime/runtime.go

@@ -209,7 +209,10 @@ func Call(address libcommon.Address, input []byte, cfg *Config) ([]byte, uint64,
 
 	vmenv := NewEnv(cfg)
 
-	sender := cfg.State.GetOrNewStateObject(cfg.Origin)
+	// Hack for firehose
+	// we don't want to track the origin account creation
+	// sender := cfg.State.GetOrNewStateObject(cfg.Origin)
+	sender := vm.AccountRef(cfg.Origin)
 	statedb := cfg.State
 	rules := vmenv.ChainRules()
 	if cfg.EVMConfig.Tracer != nil && cfg.EVMConfig.Tracer.OnTxStart != nil {

eth/tracers/internal/tracetest/firehose/blockchain_test.go

@@ -0,0 +1,81 @@
+package firehose_test
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/core"
+	"github.com/ledgerwatch/erigon/core/state"
+	"github.com/ledgerwatch/erigon/core/tracing"
+	"github.com/ledgerwatch/erigon/core/types"
+	"github.com/ledgerwatch/erigon/core/vm"
+	"github.com/ledgerwatch/erigon/tests"
+	"github.com/ledgerwatch/erigon/turbo/stages/mock"
+	"github.com/ledgerwatch/log/v3"
+	"github.com/stretchr/testify/require"
+)
+
+func runPrestateBlock(t *testing.T, prestatePath string, hooks *tracing.Hooks) {
+	t.Helper()
+
+	prestate := readPrestateData(t, prestatePath)
+	tx, err := types.UnmarshalTransactionFromBinary(common.FromHex(prestate.Input), false)
+	if err != nil {
+		t.Fatalf("failed to parse testcase input: %v", err)
+	}
+
+	context, err := prestate.Context.toBlockContext(prestate.Genesis)
+	require.NoError(t, err)
+	rules := prestate.Genesis.Config.Rules(context.BlockNumber, context.Time)
+
+	m := mock.Mock(t)
+	dbTx, err := m.DB.BeginRw(m.Ctx)
+	require.NoError(t, err)
+	defer dbTx.Rollback()
+	stateDB, _ := tests.MakePreState(rules, dbTx, prestate.Genesis.Alloc, uint64(context.BlockNumber), m.HistoryV3)
+
+	var logger = log.New("test")
+	genesisBlock, _, err := core.GenesisToBlock(prestate.Genesis, "", logger, nil)
+	require.NoError(t, err)
+
+	block := types.NewBlock(&types.Header{
+		ParentHash:            genesisBlock.Hash(),
+		Number:                big.NewInt(int64(context.BlockNumber)),
+		Difficulty:            context.Difficulty,
+		Coinbase:              context.Coinbase,
+		Time:                  context.Time,
+		GasLimit:              context.GasLimit,
+		BaseFee:               context.BaseFee.ToBig(),
+		ParentBeaconBlockRoot: ptr(common.Hash{}),
+	}, []types.Transaction{tx}, nil, nil, nil)
+
+	stateDB.SetLogger(hooks)
+	stateDB.SetTxContext(tx.Hash(), block.Hash(), 0)
+
+	hooks.OnBlockchainInit(prestate.Genesis.Config)
+	hooks.OnBlockStart(tracing.BlockEvent{
+		Block: block,
+		TD:    prestate.TotalDifficulty,
+	})
+
+	usedGas := uint64(0)
+	usedBlobGas := uint64(0)
+	_, _, err = core.ApplyTransaction(
+		prestate.Genesis.Config,
+		nil,
+		nil,
+		&context.Coinbase,
+		new(core.GasPool).AddGas(block.GasLimit()),
+		stateDB,
+		state.NewNoopWriter(),
+		block.Header(),
+		tx,
+		&usedGas,
+		&usedBlobGas,
+		vm.Config{Tracer: hooks},
+	)
+	require.NoError(t, err)
+
+	hooks.OnBlockEnd(nil)
+}

eth/tracers/internal/tracetest/firehose/firehose_test.go

@@ -0,0 +1,39 @@
+package firehose_test
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/ledgerwatch/erigon/eth/tracers/live"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFirehosePrestate(t *testing.T) {
+	testFolders := []string{
+		// "./testdata/TestFirehosePrestate/keccak256_too_few_memory_bytes_get_padded",
+		"./testdata/TestFirehosePrestate/keccak256_wrong_diff",
+	}
+
+	for _, folder := range testFolders {
+		name := filepath.Base(folder)
+
+		t.Run(name, func(t *testing.T) {
+			tracer, err := live.NewFirehoseFromRawJSON(json.RawMessage(`{
+				"applyBackwardsCompatibility": true,
+				"_private": {
+					"flushToTestBuffer": true
+				}
+			}`))
+			require.NoError(t, err)
+
+			runPrestateBlock(t, filepath.Join(folder, "prestate.json"), live.NewTracingHooksFromFirehose(tracer))
+			genesisLine, blockLines, unknownLines := readTracerFirehoseLines(t, tracer)
+			require.Len(t, unknownLines, 0, "Lines:\n%s", strings.Join(slicesMap(unknownLines, func(l unknwonLine) string { return "- '" + string(l) + "'" }), "\n"))
+			require.NotNil(t, genesisLine)
+			blockLines.assertOnlyBlockEquals(t, folder, 1)
+		})
+	}
+
+}

eth/tracers/internal/tracetest/firehose/helper_test.go

@@ -0,0 +1,159 @@
+package firehose_test
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ledgerwatch/erigon/eth/tracers/live"
+	pbeth "github.com/ledgerwatch/erigon/pb/sf/ethereum/type/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+)
+
+type firehoseInitLine struct {
+	ProtocolVersion string
+	NodeName        string
+	NodeVersion     string
+}
+
+type firehoseBlockLines []firehoseBlockLine
+
+func (lines firehoseBlockLines) assertEquals(t *testing.T, goldenDir string, expected ...firehoseBlockLineParams) {
+	actualParams := slicesMap(lines, func(l firehoseBlockLine) firehoseBlockLineParams { return l.Params })
+	require.Equal(t, expected, actualParams, "Actual lines block params do not match expected lines block params")
+
+	lines.assertOnlyBlockEquals(t, goldenDir, len(expected))
+}
+
+func (lines firehoseBlockLines) assertOnlyBlockEquals(t *testing.T, goldenDir string, expectedBlockCount int) {
+	t.Helper()
+
+	require.Len(t, lines, expectedBlockCount, "Expected %d blocks, got %d", expectedBlockCount, len(lines))
+	goldenUpdate := os.Getenv("GOLDEN_UPDATE") == "true"
+
+	for _, line := range lines {
+		goldenPath := filepath.Join(goldenDir, fmt.Sprintf("block.%d.golden.json", line.Block.Header.Number))
+		if !goldenUpdate && !fileExists(t, goldenPath) {
+			t.Fatalf("the golden file %q does not exist, re-run with 'GOLDEN_UPDATE=true go test ./... -run %q' to generate the intial version", goldenPath, t.Name())
+		}
+
+		content, err := protojson.MarshalOptions{Indent: "  "}.Marshal(line.Block)
+		require.NoError(t, err)
+
+		if goldenUpdate {
+			require.NoError(t, os.MkdirAll(filepath.Dir(goldenPath), 0755))
+			require.NoError(t, os.WriteFile(goldenPath, content, 0644))
+		}
+
+		expected, err := os.ReadFile(goldenPath)
+		require.NoError(t, err)
+
+		expectedBlock := &pbeth.Block{}
+		require.NoError(t, protojson.Unmarshal(expected, expectedBlock))
+
+		if !proto.Equal(expectedBlock, line.Block) {
+			assert.EqualExportedValues(t, expectedBlock, line.Block, "Run 'GOLDEN_UPDATE=true go test ./... -run %q' to update golden file", t.Name())
+		}
+	}
+}
+
+func fileExists(t *testing.T, path string) bool {
+	t.Helper()
+	stat, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+
+		t.Fatal(err)
+	}
+
+	return !stat.IsDir()
+}
+
+func slicesMap[T any, U any](s []T, f func(T) U) []U {
+	result := make([]U, len(s))
+	for i, v := range s {
+		result[i] = f(v)
+	}
+	return result
+}
+
+type firehoseBlockLine struct {
+	// We split params and block to make it easier to compare stuff
+	Params firehoseBlockLineParams
+	Block  *pbeth.Block
+}
+
+type firehoseBlockLineParams struct {
+	Number       string
+	Hash         string
+	PreviousNum  string
+	PreviousHash string
+	LibNum       string
+	Time         string
+}
+
+type unknwonLine string
+
+func readTracerFirehoseLines(t *testing.T, tracer *live.Firehose) (genesisLine *firehoseInitLine, blockLines firehoseBlockLines, unknownLines []unknwonLine) {
+	t.Helper()
+
+	lines := bytes.Split(tracer.InternalTestingBuffer().Bytes(), []byte{'\n'})
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+
+		parts := bytes.Split(line, []byte{' '})
+		if len(parts) == 0 || string(parts[0]) != "FIRE" {
+			unknownLines = append(unknownLines, unknwonLine(line))
+			continue
+		}
+
+		action := string(parts[1])
+		fireParts := parts[2:]
+		switch action {
+		case "INIT":
+			genesisLine = &firehoseInitLine{
+				ProtocolVersion: string(fireParts[0]),
+				NodeName:        string(fireParts[1]),
+				NodeVersion:     string(fireParts[2]),
+			}
+
+		case "BLOCK":
+			protoBytes, err := base64.StdEncoding.DecodeString(string(fireParts[6]))
+			require.NoError(t, err)
+
+			block := &pbeth.Block{}
+			require.NoError(t, proto.Unmarshal(protoBytes, block))
+
+			blockLines = append(blockLines, firehoseBlockLine{
+				Params: firehoseBlockLineParams{
+					Number:       string(fireParts[0]),
+					Hash:         string(fireParts[1]),
+					PreviousNum:  string(fireParts[2]),
+					PreviousHash: string(fireParts[3]),
+					LibNum:       string(fireParts[4]),
+					Time:         string(fireParts[5]),
+				},
+				Block: block,
+			})
+
+		default:
+			unknownLines = append(unknownLines, unknwonLine(line))
+		}
+	}
+
+	return
+}
+
+func ptr[T any](v T) *T {
+	return &v
+}