diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index f947e4bc9bae..33ba9f0b8771 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/internal/utesting" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" ) @@ -530,7 +529,7 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error { for i, key := range hashes { keys[i] = common.CopyBytes(key[:]) } - nodes := make(light.NodeList, len(proof)) + nodes := make(trie.NodeList, len(proof)) for i, node := range proof { nodes[i] = node } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index b2fd03766eca..2ee65f76e980 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" @@ -321,7 +320,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac it.Release() // Generate the Merkle proofs for the first and last account - proof := light.NewNodeSet() + proof := trie.NewNodeSet() if err := tr.Prove(req.Origin[:], proof); err != nil { log.Warn("Failed to prove account range", "origin", req.Origin, "err", err) return nil, nil @@ -427,7 +426,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if err != nil { return nil, nil } - proof := light.NewNodeSet() + proof := trie.NewNodeSet() if err := stTrie.Prove(origin[:], proof); err != nil { log.Warn("Failed to prove storage range", "origin", req.Origin, "err", err) return nil, nil diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 0f5f2ccdfeb9..dc516c5ad45d 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/msgrate" "github.com/ethereum/go-ethereum/rlp" @@ -2394,7 +2393,7 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco for i, key := range hashes { keys[i] = common.CopyBytes(key[:]) } - nodes := make(light.NodeList, len(proof)) + nodes := make(trie.NodeList, len(proof)) for i, node := range proof { nodes[i] = node } @@ -2639,7 +2638,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo for j, key := range hashes[i] { keys[j] = common.CopyBytes(key[:]) } - nodes := make(light.NodeList, 0, len(proof)) + nodes := make(trie.NodeList, 0, len(proof)) if i == len(hashes)-1 { for _, node := range proof { nodes = append(nodes, node) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 1514ad4e1344..8f01a33e1d54 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -273,7 +272,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H // Unless we send the entire trie, we need to supply proofs // Actually, we need to supply proofs either way! This seems to be an implementation // quirk in go-ethereum - proof := light.NewNodeSet() + proof := trie.NewNodeSet() if err := t.accountTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove inexistence of origin", "origin", origin, "error", err) } @@ -353,7 +352,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm if originHash != (common.Hash{}) || (abort && len(keys) > 0) { // If we're aborting, we need to prove the first and last item // This terminates the response (and thus the loop) - proof := light.NewNodeSet() + proof := trie.NewNodeSet() stTrie := t.storageTries[account] // Here's a potential gotcha: when constructing the proof, we cannot @@ -411,7 +410,7 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco if exit { // If we're aborting, we need to prove the first and last item // This terminates the response (and thus the loop) - proof := light.NewNodeSet() + proof := trie.NewNodeSet() stTrie := t.storageTries[account] // Here's a potential gotcha: when constructing the proof, we cannot @@ -599,9 +598,10 @@ func testSyncBloatedProof(t *testing.T, scheme string) { vals = append(vals, entry.v) } // The proofs - proof := light.NewNodeSet() + proof := trie.NewNodeSet() if err := t.accountTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove origin", "origin", origin, "error", err) + t.logger.Error("Could not prove origin", "origin", origin, "error", err) } // The bloat: add proof of every single element for _, entry := range t.accountValues { diff --git a/les/client_handler.go b/les/client_handler.go index 4cfeba08fe14..27cf1fa878b0 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/trie" ) // clientHandler is responsible for receiving and processing all incoming server @@ -236,7 +237,7 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { p.Log().Trace("Received les/2 proofs response") var resp struct { ReqID, BV uint64 - Data light.NodeList + Data trie.NodeList } if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) diff --git a/les/handler_test.go b/les/handler_test.go index 26a083f475da..e74fd6411709 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -401,7 +401,7 @@ func testGetProofs(t *testing.T, protocol int) { bc := server.handler.blockchain var proofreqs []ProofReq - proofsV2 := light.NewNodeSet() + proofsV2 := trie.NewNodeSet() accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ { @@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { - proofsV2 := light.NewNodeSet() + proofsV2 := trie.NewNodeSet() t, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB()) t.Prove(account, proofsV2) expected = proofsV2.NodeList() diff --git a/les/odr_requests.go b/les/odr_requests.go index 2b23e0540cc0..f89a930492da 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -222,7 +222,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { if msg.MsgType != MsgProofsV2 { return errInvalidMessageType } - proofs := msg.Obj.(light.NodeList) + proofs := msg.Obj.(trie.NodeList) // Verify the proof and store if checks out nodeSet := proofs.NodeSet() reads := &readTraceDB{db: nodeSet} @@ -308,7 +308,7 @@ type HelperTrieReq struct { } type HelperTrieResps struct { // describes all responses, not just a single one - Proofs light.NodeList + Proofs trie.NodeList AuxData [][]byte } diff --git a/les/peer.go b/les/peer.go index 48381689ef7a..9cc7e382d29e 100644 --- a/les/peer.go +++ b/les/peer.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -899,7 +900,7 @@ func (p *clientPeer) replyReceiptsRLP(reqID uint64, receipts []rlp.RawValue) *re } // replyProofsV2 creates a reply with a batch of merkle proofs, corresponding to the ones requested. -func (p *clientPeer) replyProofsV2(reqID uint64, proofs light.NodeList) *reply { +func (p *clientPeer) replyProofsV2(reqID uint64, proofs trie.NodeList) *reply { data, _ := rlp.EncodeToBytes(proofs) return &reply{p.rw, ProofsV2Msg, reqID, data} } diff --git a/les/server_requests.go b/les/server_requests.go index 485be6d9e9e6..03d0097b84d8 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -378,7 +378,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { err error ) bc := backend.BlockChain() - nodes := light.NewNodeSet() + nodes := trie.NewNodeSet() for i, request := range r.Reqs { if i != 0 && !waitOrStop() { @@ -463,7 +463,7 @@ func handleGetHelperTrieProofs(msg Decoder) (serveRequestFn, uint64, uint64, err auxData [][]byte ) bc := backend.BlockChain() - nodes := light.NewNodeSet() + nodes := trie.NewNodeSet() for i, request := range r.Reqs { if i != 0 && !waitOrStop() { return nil diff --git a/light/odr.go b/light/odr.go index 2597027435ba..2007a0827054 100644 --- a/light/odr.go +++ b/light/odr.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) // NoOdr is the default context passed to an ODR capable function when the ODR @@ -90,7 +91,7 @@ func StorageTrieID(state *TrieID, address common.Address, root common.Hash) *Tri type TrieRequest struct { Id *TrieID Key []byte - Proof *NodeSet + Proof *trie.NodeSet } // StoreResult stores the retrieved data in local database @@ -143,7 +144,7 @@ type ChtRequest struct { ChtRoot common.Hash Header *types.Header Td *big.Int - Proof *NodeSet + Proof *trie.NodeSet } // StoreResult stores the retrieved data in local database @@ -163,7 +164,7 @@ type BloomRequest struct { SectionIndexList []uint64 BloomTrieRoot common.Hash BloomBits [][]byte - Proofs *NodeSet + Proofs *trie.NodeSet } // StoreResult stores the retrieved data in local database diff --git a/light/odr_test.go b/light/odr_test.go index d8a7f1067556..a20d4eba81bc 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -95,7 +95,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { if err != nil { panic(err) } - nodes := NewNodeSet() + nodes := trie.NewNodeSet() t.Prove(req.Key, nodes) req.Proof = nodes case *CodeRequest: diff --git a/light/postprocess.go b/light/postprocess.go index 13d75f8617ae..6f549b7faca4 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -363,7 +363,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { indexCh := make(chan uint, types.BloomBitLength) type res struct { - nodes *NodeSet + nodes *trie.NodeSet err error } resCh := make(chan res, types.BloomBitLength) diff --git a/light/nodeset.go b/trie/nodeset.go similarity index 99% rename from light/nodeset.go rename to trie/nodeset.go index 3662596785c7..8958dc137445 100644 --- a/light/nodeset.go +++ b/trie/nodeset.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package light +package trie import ( "errors" diff --git a/trie/stackproof.go b/trie/stackproof.go new file mode 100644 index 000000000000..8b4d4828b0c7 --- /dev/null +++ b/trie/stackproof.go @@ -0,0 +1,157 @@ +package trie + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +func nodeToStacktrie(n node, key []byte, writeFn NodeWriteFunc) *StackTrie { + st := stackTrieFromPool(writeFn, common.Hash{}) + switch n := n.(type) { + case *shortNode: + st.nodeType = extNode + st.key = append([]byte{}, n.Key...) + case *fullNode: + st.nodeType = branchNode + idx := int(key[0]) + for i := 0; i < idx; i++ { + sibling := n.Children[i] + if sibling == nil { + continue + } + siblingNode := stackTrieFromPool(writeFn, common.Hash{}) + siblingNode.nodeType = hashedNode + + if hash, ok := sibling.(hashNode); ok { + siblingNode.val = []byte(hash) + } else { + // This happens is the sibling is small enough (<32B) to be inlined, + // in which case the rlp-encoded node is embedded instead of the hash + short := sibling.(*shortNode) + short.Key = hexToCompact(short.Key) + siblingNode.val = nodeToBytes(short) + } + st.children[i] = siblingNode + } + default: + panic(fmt.Sprintf("%T", n)) + } + return st +} + +func resolveFromProof(proofDb ethdb.KeyValueReader, hash common.Hash) (node, error) { + data, _ := proofDb.Get(hash[:]) + if data == nil { + return nil, fmt.Errorf("proof node (hash %064x) missing", hash) + } + n, err := decodeNode(data[:], data) + if err != nil { + return nil, fmt.Errorf("bad proof node: %v", err) + } + return n, err +} + +// newStackTrieFromProof creates a new stacktrie, and initialises it from the given +// proof. It does so by starting at the given root, traverses along the given +// key, and, one by one, converts the nodes into stacktrie elements. +// +// OBS: The resulting stacktrie instance is not guaranteed to be structurally +// identical to a stacktrie which is initialized from scratch by feeding the +// corresponding elements! +// A proof-initialized (PI) stack-trie has some implicit prescient knowledge! Therefore, +// a PI can have already expanded a shortnode into shortnode+fullnode, which a non-PI +// will do only later. +// +// However, the two guarantees that PI gives are: +// - Identical hash, +// - Identical commit-sequence of nodes. +// +// OBS 2: The element in proof should _not_ be added again during value-filling. +// OBS 3: Proofs-of-abscence have not been fully tested. TODO @holiman +func newStackTrieFromProof(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader, writeFn NodeWriteFunc) (*StackTrie, error) { + var ( + err error + root, child, parent node + stRoot, stChild, stParent *StackTrie + keyrest []byte + ) + // First we need to resolve the root node from the proof. + if root, err = resolveFromProof(proofDb, rootHash); err != nil { + return nil, err + } + key = keybytesToHex(key) + parent = root + stRoot = nodeToStacktrie(root, key, writeFn) + stParent = stRoot + // Now we pursue the given key downwards, and populate the stacktrie too + for { + keyrest, child = get(parent, key, false) + switch cld := child.(type) { + case nil: + return nil, errors.New("no node at given path") + case hashNode: + child, err = resolveFromProof(proofDb, common.BytesToHash(cld)) + if err != nil { + return nil, err + } + case valueNode: + // The value node goes right into the child + stParent.val = common.CopyBytes(cld) + stParent.nodeType = leafNode + // remove the terminator + stParent.key = stParent.key[:len(stParent.key)-1] + return stRoot, nil + case *shortNode: + // In the case of small leaves, we might end up here with a fullnode + // whose child is an embedded *shortNode. + default: + // we don't expect fullnodes + panic(fmt.Sprintf("got %T", cld)) + } + stChild = nodeToStacktrie(child, keyrest, writeFn) // convert to stacktrie equivalent + // Link the parent and child. + switch pnode := parent.(type) { + case *shortNode: + stParent.children[0] = stChild + case *fullNode: + stParent.children[key[0]] = stChild + default: + panic(fmt.Sprintf("%T: invalid node: %v", pnode, pnode)) + } + key = keyrest + parent = child + stParent = stChild + } +} + +func (st *StackTrie) dumpTrie(lvl int) { + var indent []byte + for i := 0; i < lvl; i++ { + indent = append(indent, ' ') + } + switch st.nodeType { + case branchNode: + fmt.Printf("\n%s FN (key='%#x')", string(indent), st.key) + + for i := 0; i < 16; i++ { + if st.children[i] == nil { + continue + } + fmt.Printf("\n%s %#x. ", string(indent), i) + st.children[i].dumpTrie(lvl + 1) + } + fmt.Println("") + case extNode: + fmt.Printf("%s: sn('%#x')", string(indent), st.key) + st.children[0].dumpTrie(lvl + 1) + case leafNode: + fmt.Printf("%s: leaf('%#x'): %x ", string(indent), st.key, st.val) + case hashedNode: + fmt.Printf("hash: %#x %x", st.val, st.key) + default: + fmt.Printf("Foo: %d ? ", st.nodeType) + } +} diff --git a/trie/stackproof_test.go b/trie/stackproof_test.go new file mode 100644 index 000000000000..87b41f4eeeca --- /dev/null +++ b/trie/stackproof_test.go @@ -0,0 +1,105 @@ +package trie + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "golang.org/x/crypto/sha3" + "golang.org/x/exp/slices" +) + +func trieWithSmallValues() (*Trie, map[string]*kv) { + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) + vals := make(map[string]*kv) + // This loop creates a few dense nodes with small leafs: hence will + // cause embedded nodes. + for i := byte(0); i < 100; i++ { + value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} + trie.MustUpdate(value.k, value.v) + vals[string(value.k)] = value + } + return trie, vals +} + +func TestStRangeProofLeftside(t *testing.T) { + trie, vals := randomTrie(4096) + testStRangeProofLeftside(t, trie, vals) +} + +func TestStRangeProofLeftsideSmallValues(t *testing.T) { + trie, vals := trieWithSmallValues() + testStRangeProofLeftside(t, trie, vals) +} + +func testStRangeProofLeftside(t *testing.T, trie *Trie, vals map[string]*kv) { + var ( + want = trie.Hash() + entries []*kv + ) + for _, kv := range vals { + entries = append(entries, kv) + } + slices.SortFunc(entries, (*kv).cmp) + for start := 10; start < len(vals); start *= 2 { + // Set write-fn on both stacktries, to compare outputs + var ( + haveSponge = &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "have"} + wantSponge = &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "want"} + proof = memorydb.New() + refTrie *StackTrie + ) + + // Provide the proof for the first entry + if err := trie.Prove(entries[start].k, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + // Initiate the stacktrie with the proof + stTrie, err := newStackTrieFromProof(trie.Hash(), entries[start].k, proof, func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + rawdb.WriteTrieNode(haveSponge, owner, path, hash, blob, "path") + }) + if err != nil { + t.Fatal(err) + } + { // Initiate a reference stacktrie without proof (filling manually) + recording := false + refTrie = NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + if recording { // Avoid recording commits in the prefill stage + rawdb.WriteTrieNode(wantSponge, owner, path, hash, blob, "path") + } + }) + for i := 0; i <= start; i++ { // do prefill + k, v := common.CopyBytes(entries[i].k), common.CopyBytes(entries[i].v) + refTrie.Update(k, v) + } + recording = true + } + // Feed the remaining values into them both + for i := start + 1; i < len(vals); i++ { + stTrie.Update(entries[i].k, common.CopyBytes(entries[i].v)) + refTrie.Update(entries[i].k, common.CopyBytes(entries[i].v)) + } + // Verify the final trie hash + if have := stTrie.Hash(); have != want { + t.Fatalf("wrong hash, have %x want %x\n", have, want) + } + if have := refTrie.Hash(); have != want { + t.Fatalf("wrong hash, have %x want %x\n", have, want) + } + // Verify the sequence of committed nodes + if have, want := haveSponge.sponge.Sum(nil), wantSponge.sponge.Sum(nil); !bytes.Equal(have, want) { + // Show the journal + t.Logf("Expected:") + for i, v := range wantSponge.journal { + t.Logf("op %d: %v", i, v) + } + t.Logf("Stacktrie:") + for i, v := range haveSponge.journal { + t.Logf("op %d: %v", i, v) + } + t.Errorf("proof from %d: disk write sequence wrong:\nhave %x want %x\n", start, have, want) + } + } +}