Various fixes for rust-verkle proof format compatibility (#67)

* code to extract the block

* separate proof from keys in block

* display state root of block 0

* change file name to reflect the correct block number

* use RLP instead of flat binary for keyvals in block

* update go-verkle to fix build

* fix rebase issues

* make test pass

* fix issue in map copy

Co-authored-by: Jared Wasinger <j-wasinger@hotmail.com>

core/chain_makers.go

@@ -348,8 +348,8 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 					}
 				}
 				vtr.Hash()
-				p, err := vtr.ProveAndSerialize(keys, statedb.Witness().KeyVals())
-				block.SetVerkleProof(p)
+				p, k, err := vtr.ProveAndSerialize(keys, statedb.Witness().KeyVals())
+				block.SetVerkleProof(p, k)
 				if err != nil {
 					panic(err)
 				}

core/state_processor_test.go

@@ -17,8 +17,11 @@
 package core
 
 import (
+	"bytes"
 	"crypto/ecdsa"
+	"fmt"
 	"math/big"
+	"os"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -31,6 +34,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"golang.org/x/crypto/sha3"
 )
@@ -377,32 +381,38 @@ func TestProcessStateless(t *testing.T) {
 	genesis := gspec.MustCommit(db)
 	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
 	defer blockchain.Stop()
-	var blockGasUsedExpected uint64
-	chain, _ := GenerateVerkleChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {
+	txCost1 := params.WitnessBranchWriteCost*2 + params.WitnessBranchReadCost*2 + params.WitnessChunkWriteCost*3 + params.WitnessChunkReadCost*12 + params.TxGas
+	txCost2 := params.WitnessBranchWriteCost + params.WitnessBranchReadCost*2 + params.WitnessChunkWriteCost*2 + params.WitnessChunkReadCost*10 + params.TxGas
+	blockGasUsedExpected := txCost1 * 2 + txCost2
+	chain, _ := GenerateVerkleChain(gspec.Config, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) {
 		// TODO need to check that the tx cost provided is the exact amount used (no remaining left-over)
-		txCost := params.WitnessBranchWriteCost*2 + params.WitnessBranchReadCost*2 + params.WitnessChunkWriteCost*3 + params.WitnessChunkReadCost*12 + params.TxGas
-		blockGasUsedExpected += txCost * 2
-		tx, _ := types.SignTx(types.NewTransaction(uint64(i) * 3, common.Address{1, 2, 3}, big.NewInt(999), txCost, big.NewInt(875000000), nil), signer, testKey)
+		tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{1, 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey)
 		gen.AddTx(tx)
-		tx, _ = types.SignTx(types.NewTransaction(uint64(i) * 3 + 1, common.Address{}, big.NewInt(999), txCost, big.NewInt(875000000), nil), signer, testKey)
+		tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey)
 		gen.AddTx(tx)
-		txCost = params.WitnessBranchWriteCost + params.WitnessBranchReadCost*2 + params.WitnessChunkWriteCost*2 + params.WitnessChunkReadCost*10 + params.TxGas
-		blockGasUsedExpected += txCost
-		tx, _ = types.SignTx(types.NewTransaction(uint64(i) * 3 + 2, common.Address{}, big.NewInt(0), txCost, big.NewInt(875000000), nil), signer, testKey)
+		tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey)
 		gen.AddTx(tx)
 	})
 
+	f, _ := os.Create("block2.rlp")
+	defer f.Close()
+	var buf bytes.Buffer
+	rlp.Encode(&buf, chain[1])
+	f.Write(buf.Bytes())
+	fmt.Printf("%x\n", chain[0].Root())
+
 	_, err := blockchain.InsertChain(chain)
 	if err != nil {
 		t.Fatalf("block imported with error: %v", err)
 	}
 
-	b := blockchain.GetBlockByNumber(1)
-	if b == nil {
-		t.Fatalf("expected block 1 to be present in chain")
-	}
-
-	if b.GasUsed() != blockGasUsedExpected {
-		t.Fatalf("expected block txs to use %d, got %d\n", blockGasUsedExpected, b.GasUsed())
+	for i := 0; i < 2; i++ {
+		b := blockchain.GetBlockByNumber(uint64(i) + 1)
+		if b == nil {
+			t.Fatalf("expected block %d to be present in chain", i + 1)
+		}
+		if b.GasUsed() != blockGasUsedExpected {
+			t.Fatalf("expected block txs to use %d, got %d\n", blockGasUsedExpected, b.GasUsed())
+		}
 	}
 }

core/types/access_witness.go

@@ -69,6 +69,7 @@ func (aw *AccessWitness) SetLeafValue(addr []byte, value []byte) {
 
 	if chunk, exists := aw.Chunks[common.BytesToHash(addr)]; exists {
 		chunk.value = value
+		aw.Chunks[common.BytesToHash(addr)] = chunk
 	} else {
 		panic(fmt.Sprintf("address not in access witness: %x", addr))
 	}
@@ -204,10 +205,10 @@ func (aw *AccessWitness) Keys() [][]byte {
 	return keys
 }
 
-func (aw *AccessWitness) KeyVals() map[common.Hash][]byte {
-	result := make(map[common.Hash][]byte)
+func (aw *AccessWitness) KeyVals() map[string][]byte {
+	result := make(map[string][]byte)
 	for k, v := range aw.Chunks {
-		result[k] = v.value
+		result[string(k[:])] = v.value
 	}
 	return result
 }

core/types/block.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/gballet/go-verkle"
 )
 
 var (
@@ -87,7 +88,8 @@ type Header struct {
 	BaseFee *big.Int `json:"baseFeePerGas" rlp:"optional"`
 
 	// The verkle proof is ignored in legacy headers
-	VerkleProof []byte `json:"verkleProof" rlp:"optional"`
+	VerkleProof   []byte                `json:"verkleProof" rlp:"optional"`
+	VerkleKeyVals []verkle.KeyValuePair `json:"verkleKeyVals" rlp:"optional"`
 
 	/*
 		TODO (MariusVanDerWijden) Add this field once needed
@@ -340,8 +342,9 @@ func (b *Block) SanityCheck() error {
 	return b.header.SanityCheck()
 }
 
-func (b *Block) SetVerkleProof(vp []byte) {
+func (b *Block) SetVerkleProof(vp []byte, kv []verkle.KeyValuePair) {
 	b.header.VerkleProof = vp
+	b.header.VerkleKeyVals = kv
 }
 
 type writeCounter common.StorageSize

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
-	github.com/gballet/go-verkle v0.0.0-20220121105610-351986d619a8
+	github.com/gballet/go-verkle v0.0.0-20220128155149-95499dfcd74a
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-stack/stack v1.8.0
 	github.com/golang/protobuf v1.4.3

go.sum

@@ -163,6 +163,10 @@ github.com/gballet/go-verkle v0.0.0-20220119205306-b466d7bff5ba h1:iOhT1XFktDNN7
 github.com/gballet/go-verkle v0.0.0-20220119205306-b466d7bff5ba/go.mod h1:e1m+0UuY3AFFTubxZZ3ePf4yO/rHMAeTb7udhUAJduk=
 github.com/gballet/go-verkle v0.0.0-20220121105610-351986d619a8 h1:6fW03c2tXGttQv1b3mt9zmcXw4/srV62DltEER5mp/U=
 github.com/gballet/go-verkle v0.0.0-20220121105610-351986d619a8/go.mod h1:e1m+0UuY3AFFTubxZZ3ePf4yO/rHMAeTb7udhUAJduk=
+github.com/gballet/go-verkle v0.0.0-20220127081734-1fca5295178c h1:EoP3VKIxU+29cA4coL9TJfD8lPuaUURxbW5nHoK1joM=
+github.com/gballet/go-verkle v0.0.0-20220127081734-1fca5295178c/go.mod h1:NhFaKv4U1IBG8QkKCsKh4xBMmdori0B0eZjdywcrktE=
+github.com/gballet/go-verkle v0.0.0-20220128155149-95499dfcd74a h1:0GpdT+zyEoa8OTBtJk5EdZm7nF4HXzU+blG3io0pm5A=
+github.com/gballet/go-verkle v0.0.0-20220128155149-95499dfcd74a/go.mod h1:NhFaKv4U1IBG8QkKCsKh4xBMmdori0B0eZjdywcrktE=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

miner/worker.go

@@ -1042,11 +1042,11 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st
 	if tr := s.GetTrie(); tr.IsVerkle() {
 		vtr := tr.(*trie.VerkleTrie)
 		// Generate the proof if we are using a verkle tree
-		p, err := vtr.ProveAndSerialize(s.Witness().Keys(), s.Witness().KeyVals())
+		p, k, err := vtr.ProveAndSerialize(s.Witness().Keys(), s.Witness().KeyVals())
 		if err != nil {
 			return err
 		}
-		block.SetVerkleProof(p)
+		block.SetVerkleProof(p, k)
 	}
 	if w.isRunning() && !w.merger.TDDReached() {
 		if interval != nil {

trie/verkle.go

@@ -174,14 +174,14 @@ func (trie *VerkleTrie) IsVerkle() bool {
 	return true
 }
 
-type KeyValuePair struct {
-	Key   []byte
-	Value []byte
-}
+func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[string][]byte) ([]byte, []verkle.KeyValuePair, error) {
+	proof, _, _, _ := verkle.MakeVerkleMultiProof(trie.root, keys, kv)
+	p, kvps, err := verkle.SerializeProof(proof)
+	if err != nil {
+		return nil, nil, err
+	}
 
-func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[common.Hash][]byte) ([]byte, error) {
-	proof, _, _, _ := verkle.MakeVerkleMultiProof(trie.root, keys)
-	return verkle.SerializeProof(proof)
+	return p, kvps, nil
 }
 
 type set = map[string]struct{}