chore: replace go-merkledag walk with go-ipld-prime traversal for dag…

… export (#8506) * chore: replace go-merkledag walk with go-ipld-prime traversal for dag export This is "safe" now because we can limit duplicate block loads like go-merkledag does and won't get trapped taking a long time for complex DAGs. We can do this while we're using an exhaustive selector (like ExploreAll here) but will need an alternative strategy when we go for arbitrary selectors.
ipfs · Oct 26, 2021 · 9e8b6e5 · 9e8b6e5
1 parent acede6f
commit 9e8b6e5
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 35 deletions.
diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go
@@ -1,24 +1,26 @@
 package dagcmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"time"
 
 	"github.com/cheggaaa/pb"
+	blocks "github.com/ipfs/go-block-format"
 	cid "github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	ipld "github.com/ipfs/go-ipld-format"
-	mdag "github.com/ipfs/go-merkledag"
+	iface "github.com/ipfs/interface-go-ipfs-core"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	gocar "github.com/ipld/go-car"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 )
 
 func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
-
 	c, err := cid.Decode(req.Arguments[0])
 	if err != nil {
 		return fmt.Errorf(
@@ -32,24 +34,6 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 		return err
 	}
 
-	// Code disabled until descent-issue in go-ipld-prime is fixed
-	// https://github.com/ribasushi/gip-muddle-up
-	//
-	// sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder())
-	// car := gocar.NewSelectiveCar(
-	// 	req.Context,
-	// 	<needs to be fixed to take format.NodeGetter as well>,
-	// 	[]gocar.Dag{gocar.Dag{
-	// 		Root: c,
-	// 		Selector: sb.ExploreRecursive(
-	// 			gipselector.RecursionLimitNone(),
-	// 			sb.ExploreAll(sb.ExploreRecursiveEdge()),
-	// 		).Node(),
-	// 	}},
-	// )
-	// ...
-	// if err := car.Write(pipeW); err != nil {}
-
 	pipeR, pipeW := io.Pipe()
 
 	errCh := make(chan error, 2) // we only report the 1st error
@@ -61,15 +45,12 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
 			close(errCh)
 		}()
 
-		if err := gocar.WriteCar(
-			req.Context,
-			mdag.NewSession(
-				req.Context,
-				api.Dag(),
-			),
-			[]cid.Cid{c},
-			pipeW,
-		); err != nil {
+		store := dagStore{dag: api.Dag(), ctx: req.Context}
+		dag := gocar.Dag{Root: c, Selector: selectorparse.CommonSelector_ExploreAllRecursively}
+		// TraverseLinksOnlyOnce is safe for an exhaustive selector but won't be when we allow
+		// arbitrary selectors here
+		car := gocar.NewSelectiveCar(req.Context, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce())
+		if err := car.Write(pipeW); err != nil {
 			errCh <- err
 		}
 	}()
@@ -153,3 +134,13 @@ func finishCLIExport(res cmds.Response, re cmds.ResponseEmitter) error {
 		}
 	}
 }
+
+type dagStore struct {
+	dag iface.APIDagService
+	ctx context.Context
+}
+
+func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) {
+	obj, err := ds.dag.Get(ds.ctx, c)
+	return obj, err
+}
diff --git a/go.mod b/go.mod
@@ -58,9 +58,9 @@ require (
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.5.1
 	github.com/ipfs/tar-utils v0.0.1
-	github.com/ipld/go-car v0.3.1
+	github.com/ipld/go-car v0.3.2
 	github.com/ipld/go-codec-dagpb v1.3.0
-	github.com/ipld/go-ipld-prime v0.12.2
+	github.com/ipld/go-ipld-prime v0.12.3
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jbenet/go-temp-err-catcher v0.1.0
 	github.com/jbenet/goprocess v0.1.4

diff --git a/go.sum b/go.sum
@@ -571,16 +571,16 @@ github.com/ipfs/interface-go-ipfs-core v0.5.1 h1:1KMM7RkjUD8W5fSoRsa9xR6ZMzeL8fL
 github.com/ipfs/interface-go-ipfs-core v0.5.1/go.mod h1:lNBJrdXHtWS46evMPBdWtDQMDsrKcGbxCOGoKLkztOE=
 github.com/ipfs/tar-utils v0.0.1 h1:8Na0KBD6GddGyXwU4rXNtVTE24iuZws8mENJQPLG7W4=
 github.com/ipfs/tar-utils v0.0.1/go.mod h1:ACflm9wXvV9w0eMJt6yYXxS2zuIV+yXGNwbuq1bhLeE=
-github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs=
-github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs=
+github.com/ipld/go-car v0.3.2 h1:V9wt/80FNfbMRWSD98W5br6fyjUAyVgI2lDOTZX16Lg=
+github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE=
 github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
 github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8=
 github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA=
 github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
-github.com/ipld/go-ipld-prime v0.12.2 h1:StIquYvKIRuSEAtjJDr39fyzBtziioHPwVC75tBiXzo=
-github.com/ipld/go-ipld-prime v0.12.2/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8=
+github.com/ipld/go-ipld-prime v0.12.3 h1:furVobw7UBLQZwlEwfE26tYORy3PAK8VYSgZOSr3JMQ=
+github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=