feat(shwap): Add cache file

core/exchange.go

@@ -136,7 +136,7 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende
 	}
 
 	// extend block data
-	adder := ipld.NewProofsAdder(int(block.Data.SquareSize))
+	adder := ipld.NewProofsAdder(int(block.Data.SquareSize), false)
 	defer adder.Purge()
 
 	eds, err := extendBlock(block.Data, block.Header.Version.App, nmt.NodeVisitor(adder.VisitFn()))
@@ -181,7 +181,7 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64
 	log.Debugw("fetched signed block from core", "height", b.Header.Height)
 
 	// extend block data
-	adder := ipld.NewProofsAdder(int(b.Data.SquareSize))
+	adder := ipld.NewProofsAdder(int(b.Data.SquareSize), false)
 	defer adder.Purge()
 
 	eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn()))

core/listener.go

@@ -215,7 +215,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS
 		attribute.Int64("height", b.Header.Height),
 	)
 	// extend block data
-	adder := ipld.NewProofsAdder(int(b.Data.SquareSize))
+	adder := ipld.NewProofsAdder(int(b.Data.SquareSize), false)
 	defer adder.Purge()
 
 	eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn()))

header/headertest/fraud/testing.go

@@ -58,7 +58,7 @@ func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *eds.Store) header
 
 		hdr := *h
 		if h.Height == f.height {
-			adder := ipld.NewProofsAdder(odsSize)
+			adder := ipld.NewProofsAdder(odsSize, false)
 			square := edstest.RandByzantineEDS(f.t, odsSize, nmt.NodeVisitor(adder.VisitFn()))
 			dah, err := da.NewDataAvailabilityHeader(square)
 			require.NoError(f.t, err)

nodebuilder/store_test.go

@@ -84,7 +84,7 @@ func BenchmarkStore(b *testing.B) {
 			b.StopTimer()
 			b.ResetTimer()
 			for i := 0; i < b.N; i++ {
-				adder := ipld.NewProofsAdder(size * 2)
+				adder := ipld.NewProofsAdder(size*2, false)
 				shares := sharetest.RandShares(b, size*size)
 				eds, err := rsmt2d.ComputeExtendedDataSquare(
 					shares,

share/availability/full/availability.go

@@ -77,7 +77,7 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, header *header
 		return nil
 	}
 
-	adder := ipld.NewProofsAdder(len(dah.RowRoots))
+	adder := ipld.NewProofsAdder(len(dah.RowRoots), false)
 	ctx = ipld.CtxWithProofsAdder(ctx, adder)
 	defer adder.Purge()
 

share/eds/byzantine/share_proof.go

@@ -3,10 +3,8 @@ package byzantine
 import (
 	"context"
 	"errors"
-	"math"
 
 	"github.com/ipfs/boxo/blockservice"
-	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 
 	"github.com/celestiaorg/nmt"
@@ -87,8 +85,7 @@ func GetShareWithProof(
 	width := len(dah.RowRoots)
 	// try row proofs
 	root := dah.RowRoots[axisIdx]
-	rootCid := ipld.MustCidFromNamespacedSha256(root)
-	proof, err := getProofsAt(ctx, bGetter, rootCid, shrIdx, width)
+	proof, err := ipld.GetProof(ctx, bGetter, root, shrIdx, width)
 	if err == nil {
 		shareWithProof := &ShareWithProof{
 			Share: share,
@@ -102,8 +99,7 @@ func GetShareWithProof(
 
 	// try column proofs
 	root = dah.ColumnRoots[shrIdx]
-	rootCid = ipld.MustCidFromNamespacedSha256(root)
-	proof, err = getProofsAt(ctx, bGetter, rootCid, axisIdx, width)
+	proof, err = ipld.GetProof(ctx, bGetter, root, axisIdx, width)
 	if err != nil {
 		return nil, err
 	}
@@ -118,28 +114,6 @@ func GetShareWithProof(
 	return nil, errors.New("failed to collect proof")
 }
 
-func getProofsAt(
-	ctx context.Context,
-	bGetter blockservice.BlockGetter,
-	root cid.Cid,
-	index,
-	total int,
-) (nmt.Proof, error) {
-	proofPath := make([]cid.Cid, 0, int(math.Sqrt(float64(total))))
-	proofPath, err := ipld.GetProof(ctx, bGetter, root, proofPath, index, total)
-	if err != nil {
-		return nmt.Proof{}, err
-	}
-
-	rangeProofs := make([][]byte, 0, len(proofPath))
-	for i := len(proofPath) - 1; i >= 0; i-- {
-		node := ipld.NamespacedSha256FromCID(proofPath[i])
-		rangeProofs = append(rangeProofs, node)
-	}
-
-	return nmt.NewInclusionProof(index, index+1, rangeProofs, true), nil
-}
-
 func ProtoToShare(protoShares []*pb.Share) []*ShareWithProof {
 	shares := make([]*ShareWithProof, len(protoShares))
 	for i, share := range protoShares {

share/eds/eds.go

@@ -121,7 +121,7 @@ func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid
 
 	// this adder ignores leaves, so that they are not added to the store we iterate through in
 	// writeProofs
-	adder := ipld.NewProofsAdder(odsWidth * 2)
+	adder := ipld.NewProofsAdder(odsWidth*2, false)
 	defer adder.Purge()
 
 	eds, err := rsmt2d.ImportExtendedDataSquare(

share/ipld/get.go

@@ -157,46 +157,6 @@ func GetLeaves(ctx context.Context,
 	wg.Wait()
 }
 
-// GetProof fetches and returns the leaf's Merkle Proof.
-// It walks down the IPLD NMT tree until it reaches the leaf and returns collected proof
-func GetProof(
-	ctx context.Context,
-	bGetter blockservice.BlockGetter,
-	root cid.Cid,
-	proof []cid.Cid,
-	leaf, total int,
-) ([]cid.Cid, error) {
-	// request the node
-	nd, err := GetNode(ctx, bGetter, root)
-	if err != nil {
-		return nil, err
-	}
-	// look for links
-	lnks := nd.Links()
-	if len(lnks) == 0 {
-		p := make([]cid.Cid, len(proof))
-		copy(p, proof)
-		return p, nil
-	}
-
-	// route walk to appropriate children
-	total /= 2 // as we are using binary tree, every step decreases total leaves in a half
-	if leaf < total {
-		root = lnks[0].Cid // if target leave on the left, go with walk down the first children
-		proof = append(proof, lnks[1].Cid)
-	} else {
-		root, leaf = lnks[1].Cid, leaf-total // otherwise go down the second
-		proof, err = GetProof(ctx, bGetter, root, proof, leaf, total)
-		if err != nil {
-			return nil, err
-		}
-		return append(proof, lnks[0].Cid), nil
-	}
-
-	// recursively walk down through selected children
-	return GetProof(ctx, bGetter, root, proof, leaf, total)
-}
-
 // chanGroup implements an atomic wait group, closing a jobs chan
 // when fully done.
 type chanGroup struct {

share/ipld/namespace_data.go

@@ -14,10 +14,10 @@ import (
 	"github.com/celestiaorg/nmt"
 
 	"github.com/celestiaorg/celestia-node/share"
+	"github.com/celestiaorg/celestia-node/share/shwap"
 )
 
-var ErrNamespaceOutsideRange = errors.New("share/ipld: " +
-	"target namespace is outside of namespace range for the given root")
+var ErrNamespaceOutsideRange = shwap.ErrNamespaceOutsideRange
 
 // Option is the functional option that is applied to the NamespaceData instance
 // to configure data that needs to be stored.

share/ipld/nmt_adder.go

@@ -103,13 +103,15 @@ func BatchSize(squareSize int) int {
 
 // ProofsAdder is used to collect proof nodes, while traversing merkle tree
 type ProofsAdder struct {
-	lock   sync.RWMutex
-	proofs map[cid.Cid][]byte
+	lock          sync.RWMutex
+	collectShares bool
+	proofs        map[cid.Cid][]byte
 }
 
 // NewProofsAdder creates new instance of ProofsAdder.
-func NewProofsAdder(squareSize int) *ProofsAdder {
+func NewProofsAdder(squareSize int, collectShares bool) *ProofsAdder {
 	return &ProofsAdder{
+		collectShares: collectShares,
 		// preallocate map to fit all inner nodes for given square size
 		proofs: make(map[cid.Cid][]byte, innerNodesAmount(squareSize)),
 	}
@@ -156,7 +158,7 @@ func (a *ProofsAdder) VisitFn() nmt.NodeVisitorFn {
 	if len(a.proofs) > 0 {
 		return nil
 	}
-	return a.visitInnerNodes
+	return a.visitNodes
 }
 
 // Purge removed proofs from ProofsAdder allowing GC to collect the memory
@@ -171,10 +173,13 @@ func (a *ProofsAdder) Purge() {
 	a.proofs = nil
 }
 
-func (a *ProofsAdder) visitInnerNodes(hash []byte, children ...[]byte) {
+func (a *ProofsAdder) visitNodes(hash []byte, children ...[]byte) {
 	switch len(children) {
 	case 1:
-		break
+		if a.collectShares {
+			id := MustCidFromNamespacedSha256(hash)
+			a.addProof(id, children[0])
+		}
 	case 2:
 		id := MustCidFromNamespacedSha256(hash)
 		a.addProof(id, append(children[0], children[1]...))

share/ipld/proofs.go

@@ -0,0 +1,74 @@
+package ipld
+
+import (
+	"context"
+	"math"
+
+	"github.com/ipfs/boxo/blockservice"
+	"github.com/ipfs/go-cid"
+
+	"github.com/celestiaorg/nmt"
+)
+
+// GetProof fetches and returns the leaf's Merkle Proof.
+// It walks down the IPLD NMT tree until it reaches the leaf and returns collected proof
+func GetProof(
+	ctx context.Context,
+	bGetter blockservice.BlockGetter,
+	root []byte,
+	shareIdx,
+	total int,
+) (nmt.Proof, error) {
+	rootCid := MustCidFromNamespacedSha256(root)
+	proofPath := make([]cid.Cid, 0, int(math.Sqrt(float64(total))))
+	proofPath, err := getProof(ctx, bGetter, rootCid, proofPath, shareIdx, total)
+	if err != nil {
+		return nmt.Proof{}, err
+	}
+
+	rangeProofs := make([][]byte, 0, len(proofPath))
+	for i := len(proofPath) - 1; i >= 0; i-- {
+		node := NamespacedSha256FromCID(proofPath[i])
+		rangeProofs = append(rangeProofs, node)
+	}
+
+	return nmt.NewInclusionProof(shareIdx, shareIdx+1, rangeProofs, true), nil
+}
+
+func getProof(
+	ctx context.Context,
+	bGetter blockservice.BlockGetter,
+	root cid.Cid,
+	proof []cid.Cid,
+	leaf, total int,
+) ([]cid.Cid, error) {
+	// request the node
+	nd, err := GetNode(ctx, bGetter, root)
+	if err != nil {
+		return nil, err
+	}
+	// look for links
+	lnks := nd.Links()
+	if len(lnks) == 0 {
+		p := make([]cid.Cid, len(proof))
+		copy(p, proof)
+		return p, nil
+	}
+
+	// route walk to appropriate children
+	total /= 2 // as we are using binary tree, every step decreases total leaves in a half
+	if leaf < total {
+		root = lnks[0].Cid // if target leave on the left, go with walk down the first children
+		proof = append(proof, lnks[1].Cid)
+	} else {
+		root, leaf = lnks[1].Cid, leaf-total // otherwise go down the second
+		proof, err = getProof(ctx, bGetter, root, proof, leaf, total)
+		if err != nil {
+			return nil, err
+		}
+		return append(proof, lnks[0].Cid), nil
+	}
+
+	// recursively walk down through selected children
+	return getProof(ctx, bGetter, root, proof, leaf, total)
+}

share/eds/byzantine/share_proof_test.go → share/ipld/proofs_test.go

@@ -1,4 +1,4 @@
-package byzantine
+package ipld
 
 import (
 	"context"
@@ -11,19 +11,19 @@ import (
 	"github.com/celestiaorg/rsmt2d"
 
 	"github.com/celestiaorg/celestia-node/share"
-	"github.com/celestiaorg/celestia-node/share/ipld"
 	"github.com/celestiaorg/celestia-node/share/sharetest"
+	"github.com/celestiaorg/celestia-node/share/shwap"
 )
 
 func TestGetProof(t *testing.T) {
 	const width = 8
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
 	defer cancel()
-	bServ := ipld.NewMemBlockservice()
+	bServ := NewMemBlockservice()
 
 	shares := sharetest.RandShares(t, width*width)
-	in, err := ipld.AddShares(ctx, shares, bServ)
+	in, err := AddShares(ctx, shares, bServ)
 	require.NoError(t, err)
 
 	dah, err := da.NewDataAvailabilityHeader(in)
@@ -38,25 +38,28 @@ func TestGetProof(t *testing.T) {
 			roots = dah.ColumnRoots
 		}
 		for axisIdx := 0; axisIdx < width*2; axisIdx++ {
-			rootCid := ipld.MustCidFromNamespacedSha256(roots[axisIdx])
+			root := roots[axisIdx]
 			for shrIdx := 0; shrIdx < width*2; shrIdx++ {
-				proof, err := getProofsAt(ctx, bServ, rootCid, shrIdx, int(in.Width()))
+				proof, err := GetProof(ctx, bServ, root, shrIdx, int(in.Width()))
 				require.NoError(t, err)
-				node, err := ipld.GetLeaf(ctx, bServ, rootCid, shrIdx, int(in.Width()))
+				rootCid := MustCidFromNamespacedSha256(root)
+				node, err := GetLeaf(ctx, bServ, rootCid, shrIdx, int(in.Width()))
 				require.NoError(t, err)
-				inclusion := &ShareWithProof{
-					Share: share.GetData(node.RawData()),
-					Proof: &proof,
-					Axis:  proofType,
+
+				sample := shwap.Sample{
+					Share:     share.GetData(node.RawData()),
+					Proof:     &proof,
+					ProofType: proofType,
 				}
-				require.True(t, inclusion.Validate(&dah, proofType, axisIdx, shrIdx))
-				// swap axis indexes to test if validation still works against the orthogonal coordinate
+				var rowIdx, colIdx int
 				switch proofType {
 				case rsmt2d.Row:
-					require.True(t, inclusion.Validate(&dah, rsmt2d.Col, shrIdx, axisIdx))
+					rowIdx, colIdx = axisIdx, shrIdx
 				case rsmt2d.Col:
-					require.True(t, inclusion.Validate(&dah, rsmt2d.Row, shrIdx, axisIdx))
+					rowIdx, colIdx = shrIdx, axisIdx
 				}
+				err = sample.Validate(&dah, rowIdx, colIdx)
+				require.NoError(t, err)
 			}
 		}
 	}