verkle proof deserialization (#61)

* use proof serialization

* remove cruft

* save current state

* fix most issues up to this point

* fix remaining build issues

* update the go.mod to use the right branch

* remove custom-defined set type

* update go-verkle to get merged PRs

* extract key, value data from proof

* only activate precomp calculations if this is a verkle chain

Co-authored-by: Jared Wasinger <j-wasinger@hotmail.com>

core/chain_makers.go

@@ -347,8 +347,8 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 					}
 				}
 				vtr.Hash()
-				_, err := vtr.ProveAndSerialize(keys, statedb.Witness().KeyVals())
-				//block.SetVerkleProof(p)
+				p, err := vtr.ProveAndSerialize(keys, statedb.Witness().KeyVals())
+				block.SetVerkleProof(p)
 				if err != nil {
 					panic(err)
 				}

eth/backend.go

@@ -56,6 +56,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/gballet/go-verkle"
 )
 
 // Config contains the configuration options of the ETH protocol.
@@ -139,6 +140,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	}
 	log.Info("Initialised chain configuration", "config", chainConfig)
 
+	// Start the precomputation of Lagrange points
+	// if this config supports verkle trees.
+	if chainConfig.CancunBlock != nil {
+		log.Info("Detected the use of verkle trees, rebuilding the cache")
+		verkle.GetConfig()
+	}
+
 	if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil {
 		log.Error("Failed to recover state", "error", err)
 	}

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/cespare/cp v0.1.0
 	github.com/cloudflare/cloudflare-go v0.14.0
 	github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f
-	github.com/crate-crypto/go-ipa v0.0.0-20211223165939-ab3f49447206
+	github.com/crate-crypto/go-ipa v0.0.0-20220120174240-fe21866d2ad5
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
@@ -26,7 +26,7 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
-	github.com/gballet/go-verkle v0.0.0-20211221174846-e7cddd97495c
+	github.com/gballet/go-verkle v0.0.0-20220121105610-351986d619a8
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-stack/stack v1.8.0
 	github.com/golang/protobuf v1.4.3
@@ -66,7 +66,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 	golang.org/x/text v0.3.6
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce

go.sum

@@ -109,6 +109,10 @@ github.com/crate-crypto/go-ipa v0.0.0-20211211194659-8730b6787450 h1:FW8SSFr58vg
 github.com/crate-crypto/go-ipa v0.0.0-20211211194659-8730b6787450/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
 github.com/crate-crypto/go-ipa v0.0.0-20211223165939-ab3f49447206 h1:nMTTM1b+4WtWrb43nmTUV/FJz7M0M2g5fioLmd4QzUQ=
 github.com/crate-crypto/go-ipa v0.0.0-20211223165939-ab3f49447206/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
+github.com/crate-crypto/go-ipa v0.0.0-20220114181434-991b62f9b1da h1:2luwsOSyUPVE67DmD7s2nKM+JbxaRuOCu6Y82qBTdnI=
+github.com/crate-crypto/go-ipa v0.0.0-20220114181434-991b62f9b1da/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
+github.com/crate-crypto/go-ipa v0.0.0-20220120174240-fe21866d2ad5 h1:BsLconJDsODyRO72xjBHhxZKPCuY/kBgZdPeV4GFNlU=
+github.com/crate-crypto/go-ipa v0.0.0-20220120174240-fe21866d2ad5/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
@@ -149,6 +153,16 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG
 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
 github.com/gballet/go-verkle v0.0.0-20211221174846-e7cddd97495c h1:d34wNrUBGD6s2OCWQd31nqjkPWt+YbXUm8FAJ+7+tHM=
 github.com/gballet/go-verkle v0.0.0-20211221174846-e7cddd97495c/go.mod h1:6hL9IP6wLByFGqj1VDbruVAjfa8DghWD96o8kuNMDCA=
+github.com/gballet/go-verkle v0.0.0-20220113152809-0c00dbeef550 h1:uyv9x+hIyWF6Jrlz9pg/PtIhQS/lPGwVl5Rkp04m/Ik=
+github.com/gballet/go-verkle v0.0.0-20220113152809-0c00dbeef550/go.mod h1:2ObZwG6QCVAzvnUrsezVfjEL28O+e1cSA9xV/PB7IME=
+github.com/gballet/go-verkle v0.0.0-20220119102938-379d70afb801 h1:XoZ7eOVOdDXeVz05t3hO4JtRWXYvZa8lGm9zoPhSyEc=
+github.com/gballet/go-verkle v0.0.0-20220119102938-379d70afb801/go.mod h1:e1m+0UuY3AFFTubxZZ3ePf4yO/rHMAeTb7udhUAJduk=
+github.com/gballet/go-verkle v0.0.0-20220119112812-69e1c35fb0e0 h1:OQ2MGbAcc9X/51/llMp1CozPwc3ryRyXSQ19E2mbI2k=
+github.com/gballet/go-verkle v0.0.0-20220119112812-69e1c35fb0e0/go.mod h1:e1m+0UuY3AFFTubxZZ3ePf4yO/rHMAeTb7udhUAJduk=
+github.com/gballet/go-verkle v0.0.0-20220119205306-b466d7bff5ba h1:iOhT1XFktDNN72Ly2GUI/UTj2dnA1qI7+kyYqdpSCCk=
+github.com/gballet/go-verkle v0.0.0-20220119205306-b466d7bff5ba/go.mod h1:e1m+0UuY3AFFTubxZZ3ePf4yO/rHMAeTb7udhUAJduk=
+github.com/gballet/go-verkle v0.0.0-20220121105610-351986d619a8 h1:6fW03c2tXGttQv1b3mt9zmcXw4/srV62DltEER5mp/U=
+github.com/gballet/go-verkle v0.0.0-20220121105610-351986d619a8/go.mod h1:e1m+0UuY3AFFTubxZZ3ePf4yO/rHMAeTb7udhUAJduk=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -547,6 +561,8 @@ golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

trie/verkle.go

@@ -24,7 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie/utils"
 	"github.com/gballet/go-verkle"
 )
@@ -180,33 +179,20 @@ type KeyValuePair struct {
 	Value []byte
 }
 
-type verkleproof struct {
-	Proof *verkle.Proof
+func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[common.Hash][]byte) ([]byte, error) {
+	proof, _, _, _ := verkle.MakeVerkleMultiProof(trie.root, keys)
+	return verkle.SerializeProof(proof)
+}
 
-	Cis     []*verkle.Point
-	Indices []byte
-	Yis     []*verkle.Fr
+type set = map[string]struct{}
 
-	Leaves []KeyValuePair
+func hasKey(s set, key []byte) bool {
+	_, ok := s[string(key)]
+	return ok
 }
 
-func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[common.Hash][]byte) ([]byte, error) {
-	proof, cis, indices, yis := verkle.MakeVerkleMultiProof(trie.root, keys)
-	vp := verkleproof{
-		Proof:   proof,
-		Cis:     cis,
-		Indices: indices,
-		Yis:     yis,
-	}
-	for key, val := range kv {
-		var k [32]byte
-		copy(k[:], key[:])
-		vp.Leaves = append(vp.Leaves, KeyValuePair{
-			Key:   k[:],
-			Value: val,
-		})
-	}
-	return rlp.EncodeToBytes(vp)
+func addKey(s set, key []byte) {
+	s[string(key)] = struct{}{}
 }
 
 func DeserializeAndVerifyVerkleProof(serialized []byte) (map[common.Hash]common.Hash, error) {
@@ -221,17 +207,89 @@ func DeserializeAndVerifyVerkleProof(serialized []byte) (map[common.Hash]common.
 	return leaves, nil
 }
 
-func deserializeVerkleProof(proof []byte) (*verkle.Proof, []*verkle.Point, []byte, []*verkle.Fr, map[common.Hash]common.Hash, error) {
-	var vp verkleproof
-	err := rlp.DecodeBytes(proof, &vp)
+func deserializeVerkleProof(serialized []byte) (*verkle.Proof, []*verkle.Point, []byte, []*verkle.Fr, map[common.Hash]common.Hash, error) {
+	var (
+		indices           []byte       // List of zis
+		yis               []*verkle.Fr // List of yis
+		seenIdx, seenComm set          // Mark when a zi/yi has already been seen in deserialization
+		others            set          // Mark when an "other" stem has been seen
+	)
+
+	proof, err := verkle.DeserializeProof(serialized)
 	if err != nil {
 		return nil, nil, nil, nil, nil, fmt.Errorf("verkle proof deserialization error: %w", err)
 	}
-	leaves := make(map[common.Hash]common.Hash, len(vp.Leaves))
-	for _, kvp := range vp.Leaves {
-		leaves[common.BytesToHash(kvp.Key)] = common.BytesToHash(kvp.Value)
+
+	for _, stem := range proof.PoaStems {
+		addKey(others, stem)
+	}
+
+	keyvals := make(map[common.Hash]common.Hash)
+	for i, key := range proof.Keys {
+		keyvals[common.BytesToHash(key)] = common.BytesToHash(proof.Values[i])
+	}
+
+	if len(proof.Keys) != len(proof.Values) {
+		return nil, nil, nil, nil, nil, fmt.Errorf("keys and values are of different length %d != %d", len(proof.Keys), len(proof.Values))
+	}
+	if len(proof.Keys) != len(proof.ExtStatus) {
+		return nil, nil, nil, nil, nil, fmt.Errorf("keys and values are of different length %d != %d", len(proof.Keys), len(proof.Values))
 	}
-	return vp.Proof, vp.Cis, vp.Indices, vp.Yis, leaves, nil
+
+	// Rebuild the tree, creating nodes in the lexicographic order of their path
+	lastcomm, lastpoa := 0, 0
+	root := verkle.NewStateless()
+	for i, es := range proof.ExtStatus {
+		depth := es & 0x1F
+		status := es >> 5
+		node := root
+		stem := proof.Keys[i]
+
+		// go over the stem's bytes, in order to rebuild the internal nodes
+		for j := byte(0); j < depth; j++ {
+			// Recurse into the tree that is being rebuilt
+			if node.Children()[stem[j]] == nil {
+				node.SetChild(int(stem[j]), verkle.NewStatelessWithCommitment(proof.Cs[lastcomm]))
+				lastcomm++
+			}
+
+			node = node.Children()[stem[j]].(*verkle.StatelessNode)
+
+			// if that zi hasn't been encountered yet, add it to
+			// the list of zis sorted by path.
+			if !hasKey(seenIdx, stem[:j]) {
+				addKey(seenIdx, stem[:j])
+				indices = append(indices, stem[j])
+			}
+
+			// same thing with a yi
+			if !hasKey(seenComm, stem[:j]) {
+				addKey(seenComm, stem[:j])
+				yis = append(yis, node.ComputeCommitment())
+			}
+		}
+
+		// Reached the end, add the extension-and-suffix tree
+		switch status {
+		case 0:
+			// missing stem, leave it as is
+			break
+		case 1:
+			// another stem is found, build it
+			node.SetStem(proof.PoaStems[lastpoa])
+			lastpoa++
+			break
+		case 2:
+			// stem is present
+			node.SetStem(stem[:31])
+			break
+		default:
+			return nil, nil, nil, nil, nil, fmt.Errorf("verkle proof deserialization error: invalid extension status %d", status)
+		}
+
+	}
+
+	return proof, proof.Cs, indices, yis, keyvals, nil
 }
 
 // Copy the values here so as to avoid an import cycle

trie/verkle_iterator.go

@@ -105,7 +105,7 @@ func (it *verkleNodeIterator) Next(descend bool) bool {
 		if err != nil {
 			panic(err)
 		}
-		it.current, err = verkle.ParseNode(data, len(it.stack)-1)
+		it.current, err = verkle.ParseNode(data, byte(len(it.stack)-1))
 		if err != nil {
 			panic(err)
 		}