Merge remote-tracking branch 'upstream/devel' into devel+classic

cmd/hack/db/lmdb.go

@@ -577,7 +577,7 @@ func generate6(_ kv.RwDB, tx kv.RwTx) (bool, error) {
 }
 
 func dropT(_ kv.RwDB, tx kv.RwTx) (bool, error) {
-	if err := tx.(kv.BucketMigrator).ClearBucket("t"); err != nil {
+	if err := tx.ClearBucket("t"); err != nil {
 		return false, err
 	}
 	return true, nil
@@ -607,14 +607,14 @@ func generate7(_ kv.RwDB, tx kv.RwTx) (bool, error) {
 }
 
 func dropT1(_ kv.RwDB, tx kv.RwTx) (bool, error) {
-	if err := tx.(kv.BucketMigrator).ClearBucket("t1"); err != nil {
+	if err := tx.ClearBucket("t1"); err != nil {
 		return false, err
 	}
 	return true, nil
 }
 
 func dropT2(_ kv.RwDB, tx kv.RwTx) (bool, error) {
-	if err := tx.(kv.BucketMigrator).ClearBucket("t2"); err != nil {
+	if err := tx.ClearBucket("t2"); err != nil {
 		return false, err
 	}
 	return true, nil
@@ -624,7 +624,7 @@ func dropT2(_ kv.RwDB, tx kv.RwTx) (bool, error) {
 func generate8(_ kv.RwDB, tx kv.RwTx) (bool, error) {
 	for i := 0; i < 100; i++ {
 		k := fmt.Sprintf("table_%05d", i)
-		if err := tx.(kv.BucketMigrator).CreateBucket(k); err != nil {
+		if err := tx.CreateBucket(k); err != nil {
 			return false, err
 		}
 	}
@@ -656,7 +656,7 @@ func generate9(tx kv.RwTx, entries int) error {
 func dropAll(_ kv.RwDB, tx kv.RwTx) (bool, error) {
 	for i := 0; i < 100; i++ {
 		k := fmt.Sprintf("table_%05d", i)
-		if err := tx.(kv.BucketMigrator).DropBucket(k); err != nil {
+		if err := tx.DropBucket(k); err != nil {
 			return false, err
 		}
 	}

consensus/bor/bor.go

@@ -28,6 +28,9 @@ import (
 
 	"github.com/ledgerwatch/erigon/common"
 	"github.com/ledgerwatch/erigon/consensus"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/flags"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/whitelist"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall/span"
 	"github.com/ledgerwatch/erigon/consensus/bor/statefull"
@@ -39,9 +42,6 @@ import (
 	"github.com/ledgerwatch/erigon/core/types/accounts"
 	"github.com/ledgerwatch/erigon/crypto"
 	"github.com/ledgerwatch/erigon/crypto/cryptopool"
-	"github.com/ledgerwatch/erigon/eth/borfinality"
-	"github.com/ledgerwatch/erigon/eth/borfinality/flags"
-	"github.com/ledgerwatch/erigon/eth/borfinality/whitelist"
 	"github.com/ledgerwatch/erigon/rlp"
 	"github.com/ledgerwatch/erigon/rpc"
 	"github.com/ledgerwatch/erigon/turbo/services"
@@ -1302,7 +1302,7 @@ func (f FinalityAPIFunc) GetRootHash(start uint64, end uint64) (string, error) {
 func (c *Bor) Start(chainDB kv.RwDB) {
 	if flags.Milestone {
 		whitelist.RegisterService(c.DB)
-		borfinality.Whitelist(c.HeimdallClient, c.DB, chainDB, c.blockReader, c.logger,
+		finality.Whitelist(c.HeimdallClient, c.DB, chainDB, c.blockReader, c.logger,
 			FinalityAPIFunc(func(start uint64, end uint64) (string, error) {
 				ctx := context.Background()
 				tx, err := chainDB.BeginRo(ctx)

eth/borfinality/api.go → consensus/bor/finality/api.go

@@ -1,11 +1,11 @@
-package borfinality
+package finality
 
 import (
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/whitelist"
 	"github.com/ledgerwatch/erigon/core/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/borfinality/whitelist"
 )
 
 func GetFinalizedBlockNumber(tx kv.Tx) uint64 {
@@ -29,6 +29,7 @@ func GetFinalizedBlockNumber(tx kv.Tx) uint64 {
 
 	doExist, number, hash = service.GetWhitelistedCheckpoint()
 	if doExist && number <= currentBlockNum.Number.Uint64() {
+
 		blockHeader := rawdb.ReadHeaderByNumber(tx, number)
 
 		if blockHeader == nil {

eth/borfinality/bor_checkpoint_verifier.go → consensus/bor/finality/bor_verifier.go

@@ -1,14 +1,14 @@
 // nolint
-package borfinality
+package finality
 
 import (
 	"context"
 	"errors"
 	"fmt"
 
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/generics"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/whitelist"
 	"github.com/ledgerwatch/erigon/core/rawdb"
-	"github.com/ledgerwatch/erigon/eth/borfinality/generics"
-	"github.com/ledgerwatch/erigon/eth/borfinality/whitelist"
 	"github.com/ledgerwatch/erigon/metrics"
 	"github.com/ledgerwatch/log/v3"
 )
@@ -139,6 +139,10 @@ func borVerify(ctx context.Context, config *config, start uint64, end uint64, ha
 		log.Debug("Failed to get end block hash while whitelisting", "err", err)
 		return hash, errEndBlock
 	}
+	if block == nil {
+		log.Debug("Current header behind the end block", "block", end)
+		return hash, errEndBlock
+	}
 
 	hash = fmt.Sprintf("%v", block.Hash())
 

eth/borfinality/rawdb/milestone.go → consensus/bor/finality/rawdb/milestone.go

@@ -7,7 +7,7 @@ import (
 
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
-	"github.com/ledgerwatch/erigon/eth/borfinality/generics"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/generics"
 	"github.com/ledgerwatch/log/v3"
 )
 

eth/borfinality/whitelist.go → consensus/bor/finality/whitelist.go

@@ -1,4 +1,4 @@
-package borfinality
+package finality
 
 import (
 	"context"
@@ -7,9 +7,9 @@ import (
 	"time"
 
 	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/flags"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/whitelist"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall"
-	"github.com/ledgerwatch/erigon/eth/borfinality/flags"
-	"github.com/ledgerwatch/erigon/eth/borfinality/whitelist"
 	"github.com/ledgerwatch/erigon/turbo/services"
 	"github.com/ledgerwatch/log/v3"
 )

eth/borfinality/whitelist/checkpoint.go → consensus/bor/finality/whitelist/checkpoint.go

@@ -2,8 +2,8 @@ package whitelist
 
 import (
 	"github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/borfinality/rawdb"
 	"github.com/ledgerwatch/erigon/metrics"
 )
 

eth/borfinality/whitelist/finality.go → consensus/bor/finality/whitelist/finality.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/borfinality/rawdb"
 	"github.com/ledgerwatch/log/v3"
 )
 

eth/borfinality/whitelist/milestone.go → consensus/bor/finality/whitelist/milestone.go

@@ -2,9 +2,9 @@ package whitelist
 
 import (
 	"github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/flags"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/borfinality/flags"
-	"github.com/ledgerwatch/erigon/eth/borfinality/rawdb"
 	"github.com/ledgerwatch/erigon/metrics"
 	"github.com/ledgerwatch/log/v3"
 )

eth/borfinality/whitelist/service.go → consensus/bor/finality/whitelist/service.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/borfinality/rawdb"
 )
 
 var (

eth/borfinality/whitelist/service_test.go → consensus/bor/finality/whitelist/service_test.go

@@ -12,8 +12,8 @@ import (
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/memdb"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/rawdb"
 	"github.com/ledgerwatch/erigon/core/types"
-	"github.com/ledgerwatch/erigon/eth/borfinality/rawdb"
 	"github.com/stretchr/testify/require"
 
 	"pgregory.net/rapid"

eth/borfinality/whitelist_helpers.go → consensus/bor/finality/whitelist_helpers.go

@@ -1,12 +1,12 @@
-package borfinality
+package finality
 
 import (
 	"context"
 	"errors"
 
 	"github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/whitelist"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall"
-	"github.com/ledgerwatch/erigon/eth/borfinality/whitelist"
 	"github.com/ledgerwatch/log/v3"
 )
 

consensus/bor/heimdall/heimall.go

@@ -4,10 +4,10 @@ import (
 	"context"
 
 	"github.com/ledgerwatch/erigon/consensus/bor/clerk"
+	"github.com/ledgerwatch/erigon/consensus/bor/finality/generics"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall/checkpoint"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall/milestone"
 	"github.com/ledgerwatch/erigon/consensus/bor/heimdall/span"
-	"github.com/ledgerwatch/erigon/eth/borfinality/generics"
 )
 
 func MilestoneRewindPending() bool {

core/rawdb/accessors_account.go

@@ -23,7 +23,7 @@ import (
 	"github.com/ledgerwatch/erigon/core/types/accounts"
 )
 
-func ReadAccount(db kv.Tx, addr libcommon.Address, acc *accounts.Account) (bool, error) {
+func ReadAccount(db kv.Getter, addr libcommon.Address, acc *accounts.Account) (bool, error) {
 	enc, err := db.GetOne(kv.PlainState, addr[:])
 	if err != nil {
 		return false, err

core/types/block.go

@@ -1510,28 +1510,16 @@ func (b *Block) SendersToTxs(senders []libcommon.Address) {
 // RawBody creates a RawBody based on the block. It is not very efficient, so
 // will probably be removed in favour of RawBlock. Also it panics
 func (b *Block) RawBody() *RawBody {
-	br := &RawBody{Transactions: make([][]byte, len(b.transactions)), Uncles: b.uncles, Withdrawals: b.withdrawals}
-	for i, tx := range b.transactions {
-		var err error
-		br.Transactions[i], err = rlp.EncodeToBytes(tx)
-		if err != nil {
-			panic(err)
-		}
-	}
-	return br
+	return b.Body().RawBody()
 }
 
 // RawBody creates a RawBody based on the body.
 func (b *Body) RawBody() *RawBody {
-	br := &RawBody{Transactions: make([][]byte, len(b.Transactions)), Uncles: b.Uncles, Withdrawals: b.Withdrawals}
-	for i, tx := range b.Transactions {
-		var err error
-		br.Transactions[i], err = rlp.EncodeToBytes(tx)
-		if err != nil {
-			panic(err)
-		}
+	txs, err := MarshalTransactionsBinary(b.Transactions)
+	if err != nil {
+		panic(err)
 	}
-	return br
+	return &RawBody{Transactions: txs, Uncles: b.Uncles, Withdrawals: b.Withdrawals}
 }
 
 // Size returns the true RLP encoded storage size of the block, either by encoding

core/types/block_test.go

@@ -512,3 +512,52 @@ func TestCopyTxs(t *testing.T) {
 	copies := CopyTxs(txs)
 	assert.Equal(t, txs, copies)
 }
+
+func TestRawBodyConverter(t *testing.T) {
+	blockEnc := common.FromHex("f90260f901f9a083cafc574e1f51ba9dc0568fc617a08ea2429fb384059c972f13b19fa1c8dd55a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a05fe50b260da6308036625b850b5d6ced6d0a9f814c0688bc91ffb7b7a3a54b67a0bc37d79753ad738a6dac4921e57392f145d8887476de3f783dfa7edae9283e52b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd8825208845506eb0780a0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4f861f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1c0")
+	var block Block
+	if err := rlp.DecodeBytes(blockEnc, &block); err != nil {
+		t.Fatal("decode error: ", err)
+	}
+
+	expectedTxs := [][]byte{
+		libcommon.FromHex("f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1"),
+	}
+	require.Equal(t, block.RawBody().Transactions, expectedTxs)
+}
+
+func TestRawBodyConverterShangai(t *testing.T) {
+	header := Header{
+		ParentHash: libcommon.HexToHash("0x8b00fcf1e541d371a3a1b79cc999a85cc3db5ee5637b5159646e1acd3613fd15"),
+		Coinbase:   libcommon.HexToAddress("0x571846e42308df2dad8ed792f44a8bfddf0acb4d"),
+		Root:       libcommon.HexToHash("0x351780124dae86b84998c6d4fe9a88acfb41b4856b4f2c56767b51a4e2f94dd4"),
+		Difficulty: libcommon.Big0,
+		Number:     big.NewInt(20_000_000),
+		GasLimit:   30_000_000,
+		GasUsed:    3_074_345,
+		Time:       1666343339,
+		Extra:      make([]byte, 0),
+		MixDigest:  libcommon.HexToHash("0x7f04e338b206ef863a1fad30e082bbb61571c74e135df8d1677e3f8b8171a09b"),
+		BaseFee:    big.NewInt(7_000_000_000),
+	}
+
+	withdrawals := make([]*Withdrawal, 2)
+	withdrawals[0] = &Withdrawal{
+		Index:     44555666,
+		Validator: 89,
+		Address:   libcommon.HexToAddress("0x690b9a9e9aa1c9db991c7721a92d351db4fac990"),
+		Amount:    2,
+	}
+	withdrawals[1] = &Withdrawal{
+		Index:     44555667,
+		Validator: 37,
+		Address:   libcommon.HexToAddress("0x95222290dd7278aa3ddd389cc1e1d165cc4bafe5"),
+		Amount:    5_000_000_000,
+	}
+
+	block := NewBlock(&header, nil, nil, nil, withdrawals)
+
+	rb := block.RawBody()
+
+	require.Equal(t, rb.Withdrawals, withdrawals)
+}

erigon-lib/kv/kv_interface.go

@@ -303,8 +303,6 @@ type StatelessReadTx interface {
 	// Sequence changes become visible outside the current write transaction after it is committed, and discarded on abort.
 	// Starts from 0.
 	ReadSequence(table string) (uint64, error)
-
-	BucketSize(table string) (uint64, error)
 }
 
 type StatelessWriteTx interface {
@@ -340,6 +338,16 @@ type StatelessRwTx interface {
 	StatelessWriteTx
 }
 
+// PendingMutations in-memory storage of changes
+// Later they can either be flushed to the database or abandon
+type PendingMutations interface {
+	StatelessRwTx
+	// Flush all in-memory data into `tx`
+	Flush(ctx context.Context, tx RwTx) error
+	Close()
+	BatchSize() int
+}
+
 // Tx
 // WARNING:
 //   - Tx is not threadsafe and may only be used in the goroutine that created it
@@ -397,6 +405,7 @@ type Tx interface {
 
 	// Pointer to the underlying C transaction handle (e.g. *C.MDBX_txn)
 	CHandle() unsafe.Pointer
+	BucketSize(table string) (uint64, error)
 }
 
 // RwTx

erigon-lib/kv/mdbx/kv_migrator_test.go

@@ -37,10 +37,7 @@ func TestBucketCRUD(t *testing.T) {
 
 	normalBucket := kv.ChaindataTables[15]
 	deprecatedBucket := kv.ChaindataDeprecatedTables[0]
-	migrator, ok := tx.(kv.BucketMigrator)
-	if !ok {
-		return
-	}
+	migrator := tx
 
 	// check thad buckets have unique DBI's
 	uniquness := map[kv.DBI]bool{}