diff --git a/core/commands/add.go b/core/commands/add.go
index 4eccc6aaebd..1232f1db6ab 100644
--- a/core/commands/add.go
+++ b/core/commands/add.go
@@ -149,13 +149,15 @@ remains to be implemented.
return err
}
- if !hash {
- // copy intermediary nodes from editor to our actual dagservice
- err := fileAdder.WriteOutputTo(n.DAG)
- if err != nil {
- log.Error("WRITE OUT: ", err)
- return err
- }
+ if hash {
+ return nil
+ }
+
+ // copy intermediary nodes from editor to our actual dagservice
+ _, err := fileAdder.Finalize(n.DAG)
+ if err != nil {
+ log.Error("WRITE OUT: ", err)
+ return err
}
return fileAdder.PinRoot()
diff --git a/core/commands/object.go b/core/commands/object.go
index 2b6a1494ef1..b7f129a3b7c 100644
--- a/core/commands/object.go
+++ b/core/commands/object.go
@@ -599,14 +599,17 @@ func rmLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
path := req.Arguments()[2]
- e := dagutils.NewDagEditor(nd.DAG, root)
+ e := dagutils.NewDagEditor(root, nd.DAG)
err = e.RmLink(req.Context(), path)
if err != nil {
return "", err
}
- nnode := e.GetNode()
+ nnode, err := e.Finalize(nd.DAG)
+ if err != nil {
+ return "", err
+ }
return nnode.Key()
}
@@ -636,7 +639,7 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
}
}
- e := dagutils.NewDagEditor(nd.DAG, root)
+ e := dagutils.NewDagEditor(root, nd.DAG)
childnd, err := nd.DAG.Get(req.Context(), childk)
if err != nil {
@@ -648,7 +651,10 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
return "", err
}
- nnode := e.GetNode()
+ nnode, err := e.Finalize(nd.DAG)
+ if err != nil {
+ return "", err
+ }
return nnode.Key()
}
diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go
index e46bd8523b9..c920a10f59e 100644
--- a/core/corehttp/gateway_handler.go
+++ b/core/corehttp/gateway_handler.go
@@ -320,14 +320,20 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
return
}
- e := dagutils.NewDagEditor(i.node.DAG, rnode)
+ e := dagutils.NewDagEditor(rnode, i.node.DAG)
err = e.InsertNodeAtPath(ctx, newPath, newnode, uio.NewEmptyDirectory)
if err != nil {
webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError)
return
}
- newkey, err = e.GetNode().Key()
+ nnode, err := e.Finalize(i.node.DAG)
+ if err != nil {
+ webError(w, "putHandler: could not get node", err, http.StatusInternalServerError)
+ return
+ }
+
+ newkey, err = nnode.Key()
if err != nil {
webError(w, "putHandler: could not get key of edited node", err, http.StatusInternalServerError)
return
diff --git a/core/coreunix/add.go b/core/coreunix/add.go
index 7f817f90bce..3070e874461 100644
--- a/core/coreunix/add.go
+++ b/core/coreunix/add.go
@@ -20,7 +20,7 @@ import (
"github.com/ipfs/go-ipfs/commands/files"
core "github.com/ipfs/go-ipfs/core"
- merkledag "github.com/ipfs/go-ipfs/merkledag"
+ dag "github.com/ipfs/go-ipfs/merkledag"
unixfs "github.com/ipfs/go-ipfs/unixfs"
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
)
@@ -63,7 +63,7 @@ type AddedObject struct {
}
func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) *Adder {
- e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
+ e := dagutils.NewDagEditor(newDirNode(), nil)
return &Adder{
ctx: ctx,
node: n,
@@ -90,11 +90,11 @@ type Adder struct {
Trickle bool
Wrap bool
Chunker string
- root *merkledag.Node
+ root *dag.Node
}
// Perform the actual add & pin locally, outputting results to reader
-func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
+func (params Adder) add(reader io.Reader) (*dag.Node, error) {
chnk, err := chunk.FromString(reader, params.Chunker)
if err != nil {
return nil, err
@@ -112,7 +112,7 @@ func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
)
}
-func (params *Adder) RootNode() (*merkledag.Node, error) {
+func (params *Adder) RootNode() (*dag.Node, error) {
// for memoizing
if params.root != nil {
return params.root, nil
@@ -153,8 +153,8 @@ func (params *Adder) PinRoot() error {
return params.node.Pinning.Flush()
}
-func (params *Adder) WriteOutputTo(DAG merkledag.DAGService) error {
- return params.editor.WriteOutputTo(DAG)
+func (params *Adder) Finalize(DAG dag.DAGService) (*dag.Node, error) {
+ return params.editor.Finalize(DAG)
}
// Add builds a merkledag from the a reader, pinning all objects to the local
@@ -212,7 +212,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
// to preserve the filename.
// Returns the path of the added file ("
/filename"), the DAG node of
// the directory, and and error if any.
-func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkledag.Node, error) {
+func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) {
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
dir := files.NewSliceFile("", "", []files.File{file})
fileAdder := NewAdder(n.Context(), n, nil)
@@ -230,7 +230,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkle
return gopath.Join(k.String(), filename), dagnode, nil
}
-func (params *Adder) addNode(node *merkledag.Node, path string) error {
+func (params *Adder) addNode(node *dag.Node, path string) error {
// patch it into the root
if path == "" {
key, err := node.Key()
@@ -249,7 +249,7 @@ func (params *Adder) addNode(node *merkledag.Node, path string) error {
}
// Add the given file while respecting the params.
-func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
+func (params *Adder) AddFile(file files.File) (*dag.Node, error) {
switch {
case files.IsHidden(file) && !params.Hidden:
log.Debugf("%s is hidden, skipping", file.FileName())
@@ -265,7 +265,7 @@ func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
return nil, err
}
- dagnode := &merkledag.Node{Data: sdata}
+ dagnode := &dag.Node{Data: sdata}
_, err = params.node.DAG.Add(dagnode)
if err != nil {
return nil, err
@@ -294,7 +294,7 @@ func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
return dagnode, err
}
-func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
+func (params *Adder) addDir(dir files.File) (*dag.Node, error) {
tree := newDirNode()
log.Infof("adding directory: %s", dir.FileName())
@@ -317,7 +317,7 @@ func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
_, name := gopath.Split(file.FileName())
- if err := tree.AddNodeLink(name, node); err != nil {
+ if err := tree.AddNodeLinkClean(name, node); err != nil {
return nil, err
}
}
@@ -334,7 +334,7 @@ func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
}
// outputDagnode sends dagnode info over the output channel
-func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error {
+func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
if out == nil {
return nil
}
@@ -352,20 +352,20 @@ func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error
return nil
}
-func NewMemoryDagService() merkledag.DAGService {
+func NewMemoryDagService() dag.DAGService {
// build mem-datastore for editor's intermediary nodes
bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
bsrv := bserv.New(bs, offline.Exchange(bs))
- return merkledag.NewDAGService(bsrv)
+ return dag.NewDAGService(bsrv)
}
// TODO: generalize this to more than unix-fs nodes.
-func newDirNode() *merkledag.Node {
- return &merkledag.Node{Data: unixfs.FolderPBData()}
+func newDirNode() *dag.Node {
+ return &dag.Node{Data: unixfs.FolderPBData()}
}
// from core/commands/object.go
-func getOutput(dagnode *merkledag.Node) (*Object, error) {
+func getOutput(dagnode *dag.Node) (*Object, error) {
key, err := dagnode.Key()
if err != nil {
return nil, err
diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go
index a6c6633f094..b84327dfdf3 100644
--- a/merkledag/merkledag.go
+++ b/merkledag/merkledag.go
@@ -20,6 +20,7 @@ type DAGService interface {
AddRecursive(*Node) error
Get(context.Context, key.Key) (*Node, error)
Remove(*Node) error
+ RemoveRecursive(*Node) error
// GetDAG returns, in order, all the single leve child
// nodes of the passed in node.
@@ -107,10 +108,10 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
}
// Remove deletes the given node and all of its children from the BlockService
-func (n *dagService) Remove(nd *Node) error {
+func (n *dagService) RemoveRecursive(nd *Node) error {
for _, l := range nd.Links {
if l.Node != nil {
- n.Remove(l.Node)
+ n.RemoveRecursive(l.Node)
}
}
k, err := nd.Key()
@@ -120,6 +121,14 @@ func (n *dagService) Remove(nd *Node) error {
return n.Blocks.DeleteBlock(k)
}
+func (n *dagService) Remove(nd *Node) error {
+ k, err := nd.Key()
+ if err != nil {
+ return err
+ }
+ return n.Blocks.DeleteBlock(k)
+}
+
// FetchGraph fetches all nodes that are children of the given node
func FetchGraph(ctx context.Context, root *Node, serv DAGService) error {
return EnumerateChildrenAsync(ctx, serv, root, key.NewKeySet())
diff --git a/merkledag/node.go b/merkledag/node.go
index f84695f912d..b644cae1216 100644
--- a/merkledag/node.go
+++ b/merkledag/node.go
@@ -9,6 +9,8 @@ import (
key "github.com/ipfs/go-ipfs/blocks/key"
)
+var ErrLinkNotFound = fmt.Errorf("no link by that name")
+
// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
@@ -160,7 +162,7 @@ func (n *Node) GetNodeLink(name string) (*Link, error) {
}, nil
}
}
- return nil, ErrNotFound
+ return nil, ErrLinkNotFound
}
func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*Node, error) {
diff --git a/merkledag/utils/diff.go b/merkledag/utils/diff.go
index 47ca5124f12..8ee50819c53 100644
--- a/merkledag/utils/diff.go
+++ b/merkledag/utils/diff.go
@@ -37,7 +37,7 @@ func (c *Change) String() string {
}
func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Change) (*dag.Node, error) {
- e := NewDagEditor(ds, nd)
+ e := NewDagEditor(nd, ds)
for _, c := range cs {
switch c.Type {
case Add:
@@ -71,7 +71,8 @@ func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Cha
}
}
}
- return e.GetNode(), nil
+
+ return e.Finalize(ds)
}
func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.Node) []*Change {
diff --git a/merkledag/utils/utils.go b/merkledag/utils/utils.go
index b8dde47e762..1f19e3380c3 100644
--- a/merkledag/utils/utils.go
+++ b/merkledag/utils/utils.go
@@ -4,20 +4,41 @@ import (
"errors"
"strings"
+ ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+ syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+ bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
+ bserv "github.com/ipfs/go-ipfs/blockservice"
+ offline "github.com/ipfs/go-ipfs/exchange/offline"
dag "github.com/ipfs/go-ipfs/merkledag"
)
type Editor struct {
root *dag.Node
- ds dag.DAGService
+
+ // tmp is a temporary in memory (for now) dagstore for all of the
+ // intermediary nodes to be stored in
+ tmp dag.DAGService
+
+ // src is the dagstore with *all* of the data on it, it is used to pull
+ // nodes from for modification (nil is a valid value)
+ src dag.DAGService
+}
+
+func NewMemoryDagService() dag.DAGService {
+ // build mem-datastore for editor's intermediary nodes
+ bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
+ bsrv := bserv.New(bs, offline.Exchange(bs))
+ return dag.NewDAGService(bsrv)
}
-func NewDagEditor(ds dag.DAGService, root *dag.Node) *Editor {
+// root is the node to be modified, source is the dagstore to pull nodes from (optional)
+func NewDagEditor(root *dag.Node, source dag.DAGService) *Editor {
return &Editor{
root: root,
- ds: ds,
+ tmp: NewMemoryDagService(),
+ src: source,
}
}
@@ -26,7 +47,7 @@ func (e *Editor) GetNode() *dag.Node {
}
func (e *Editor) GetDagService() dag.DAGService {
- return e.ds
+ return e.tmp
}
func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childnd *dag.Node) (*dag.Node, error) {
@@ -40,6 +61,8 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
return nil, err
}
+ _ = ds.Remove(root)
+
// ensure no link with that name already exists
_ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound
@@ -55,7 +78,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *dag.Node, create func() *dag.Node) error {
splpath := strings.Split(path, "/")
- nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
+ nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
if err != nil {
return err
}
@@ -63,33 +86,43 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *da
return nil
}
-func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
+func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
if len(path) == 1 {
- return addLink(ctx, ds, root, path[0], toinsert)
+ return addLink(ctx, e.tmp, root, path[0], toinsert)
}
- nd, err := root.GetLinkedNode(ctx, ds, path[0])
+ nd, err := root.GetLinkedNode(ctx, e.tmp, path[0])
if err != nil {
// if 'create' is true, we create directories on the way down as needed
- if err == dag.ErrNotFound && create != nil {
+ if err == dag.ErrLinkNotFound && create != nil {
nd = create()
- } else {
+ err = nil // no longer an error case
+ } else if err == dag.ErrNotFound {
+ // try finding it in our source dagstore
+ nd, err = root.GetLinkedNode(ctx, e.src, path[0])
+ }
+
+ // if we receive an ErrNotFound, then our second 'GetLinkedNode' call
+ // also fails, we want to error out
+ if err != nil {
return nil, err
}
}
- ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert, create)
+ ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create)
if err != nil {
return nil, err
}
+ _ = e.tmp.Remove(root)
+
_ = root.RemoveNodeLink(path[0])
err = root.AddNodeLinkClean(path[0], ndprime)
if err != nil {
return nil, err
}
- _, err = ds.Add(root)
+ _, err = e.tmp.Add(root)
if err != nil {
return nil, err
}
@@ -99,7 +132,7 @@ func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, pa
func (e *Editor) RmLink(ctx context.Context, path string) error {
splpath := strings.Split(path, "/")
- nd, err := rmLink(ctx, e.ds, e.root, splpath)
+ nd, err := e.rmLink(ctx, e.root, splpath)
if err != nil {
return err
}
@@ -107,7 +140,7 @@ func (e *Editor) RmLink(ctx context.Context, path string) error {
return nil
}
-func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string) (*dag.Node, error) {
+func (e *Editor) rmLink(ctx context.Context, root *dag.Node, path []string) (*dag.Node, error) {
if len(path) == 1 {
// base case, remove node in question
err := root.RemoveNodeLink(path[0])
@@ -115,7 +148,7 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return nil, err
}
- _, err = ds.Add(root)
+ _, err = e.tmp.Add(root)
if err != nil {
return nil, err
}
@@ -123,23 +156,30 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return root, nil
}
- nd, err := root.GetLinkedNode(ctx, ds, path[0])
+ // search for node in both tmp dagstore and source dagstore
+ nd, err := root.GetLinkedNode(ctx, e.tmp, path[0])
+ if err == dag.ErrNotFound {
+ nd, err = root.GetLinkedNode(ctx, e.src, path[0])
+ }
+
if err != nil {
return nil, err
}
- nnode, err := rmLink(ctx, ds, nd, path[1:])
+ nnode, err := e.rmLink(ctx, nd, path[1:])
if err != nil {
return nil, err
}
+ _ = e.tmp.Remove(root)
+
_ = root.RemoveNodeLink(path[0])
err = root.AddNodeLinkClean(path[0], nnode)
if err != nil {
return nil, err
}
- _, err = ds.Add(root)
+ _, err = e.tmp.Add(root)
if err != nil {
return nil, err
}
@@ -147,8 +187,10 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return root, nil
}
-func (e *Editor) WriteOutputTo(ds dag.DAGService) error {
- return copyDag(e.GetNode(), e.ds, ds)
+func (e *Editor) Finalize(ds dag.DAGService) (*dag.Node, error) {
+ nd := e.GetNode()
+ err := copyDag(nd, e.tmp, ds)
+ return nd, err
}
func copyDag(nd *dag.Node, from, to dag.DAGService) error {
diff --git a/merkledag/utils/utils_test.go b/merkledag/utils/utils_test.go
index 18839bf8fed..498f676b255 100644
--- a/merkledag/utils/utils_test.go
+++ b/merkledag/utils/utils_test.go
@@ -66,13 +66,12 @@ func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, path stri
}
func TestInsertNode(t *testing.T) {
- ds := mdtest.Mock()
root := new(dag.Node)
- e := NewDagEditor(ds, root)
+ e := NewDagEditor(root, nil)
testInsert(t, e, "a", "anodefortesting", false, "")
testInsert(t, e, "a/b", "data", false, "")
- testInsert(t, e, "a/b/c/d/e", "blah", false, "merkledag: not found")
+ testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name")
testInsert(t, e, "a/b/c/d/e", "foo", true, "")
testInsert(t, e, "a/b/c/d/f", "baz", true, "")
testInsert(t, e, "a/b/c/d/f", "bar", true, "")
@@ -92,7 +91,7 @@ func TestInsertNode(t *testing.T) {
func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
child := &dag.Node{Data: []byte(data)}
- ck, err := e.ds.Add(child)
+ ck, err := e.tmp.Add(child)
if err != nil {
t.Fatal(err)
}
@@ -117,8 +116,8 @@ func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr
}
if err != nil {
- t.Fatal(err)
+ t.Fatal(err, path, data, create, experr)
}
- assertNodeAtPath(t, e.ds, e.root, path, ck)
+ assertNodeAtPath(t, e.tmp, e.root, path, ck)
}
diff --git a/tar/format.go b/tar/format.go
index c0e51b028a4..fc73e17f74b 100644
--- a/tar/format.go
+++ b/tar/format.go
@@ -46,7 +46,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
root := new(dag.Node)
root.Data = []byte("ipfs/tar")
- e := dagutil.NewDagEditor(ds, root)
+ e := dagutil.NewDagEditor(root, ds)
for {
h, err := tr.Next()
@@ -91,13 +91,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
}
}
- root = e.GetNode()
- _, err = ds.Add(root)
- if err != nil {
- return nil, err
- }
-
- return root, nil
+ return e.Finalize(ds)
}
// adds a '-' to the beginning of each path element so we can use 'data' as a
@@ -178,7 +172,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
tr.hdrBuf = bytes.NewReader(headerNd.Data)
dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
- if err != nil && err != dag.ErrNotFound {
+ if err != nil && err != dag.ErrLinkNotFound {
return 0, err
}