Replace coreunix.Add with shell/fsnode's AddFromReader

cmd/ipfs/init.go

@@ -1,21 +1,22 @@
 package main
 
 import (
- "bytes"
  "errors"
  "fmt"
  "io"
+ "strings"
 
  context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
  assets "github.com/ipfs/go-ipfs/assets"
  cmds "github.com/ipfs/go-ipfs/commands"
  core "github.com/ipfs/go-ipfs/core"
- coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+ importer "github.com/ipfs/go-ipfs/importer"
+ chunk "github.com/ipfs/go-ipfs/importer/chunk"
+ merkledag "github.com/ipfs/go-ipfs/merkledag"
  namesys "github.com/ipfs/go-ipfs/namesys"
  config "github.com/ipfs/go-ipfs/repo/config"
  fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
- uio "github.com/ipfs/go-ipfs/unixfs/io"
- u "github.com/ipfs/go-ipfs/util"
+ unixfs "github.com/ipfs/go-ipfs/unixfs"
 )
 
 const nBitsForKeypairDefault = 2048
@@ -132,23 +133,24 @@ func addDefaultAssets(out io.Writer, repoRoot string) error {
  }
  defer nd.Close()
 
- dirb := uio.NewDirectory(nd.DAG)
+ dir := &merkledag.Node{Data: unixfs.FolderPBData()}
 
  // add every file in the assets pkg
  for fname, file := range assets.Init_dir {
- buf := bytes.NewBufferString(file)
- s, err := coreunix.Add(nd, buf)
+ reader := strings.NewReader(file)
+ dagNode, err := importer.BuildDagFromReader(
+ reader,
+ nd.DAG,
+ nd.Pinning.GetManual(),
+ chunk.DefaultSplitter)
  if err != nil {
  return err
  }
-
- k := u.B58KeyDecode(s)
- if err := dirb.AddChild(fname, k); err != nil {
+ if err := dir.AddNodeLink(fname, dagNode); err != nil {
  return err
  }
  }
 
- dir := dirb.GetNode()
  dkey, err := nd.DAG.Add(dir)
  if err != nil {
  return err

cmd/ipfswatch/main.go

@@ -121,12 +121,7 @@ func run(ipfsPath, watchPath string) error {
  }
  }
  proc.Go(func(p process.Process) {
- file, err := os.Open(e.Name)
- if err != nil {
- log.Println(err)
- }
- defer file.Close()
- k, err := coreunix.Add(node, file)
+ k, err := coreunix.AddR(node, e.Name)
  if err != nil {
  log.Println(err)
  }

core/corehttp/corehttp.go

@@ -23,9 +23,9 @@ var log = eventlog.Logger("core/server")
 // initially passed in if not.
 type ServeOption func(*core.IpfsNode, *http.ServeMux) (*http.ServeMux, error)
 
-// makeHandler turns a list of ServeOptions into a http.Handler that implements
+// MakeHandler turns a list of ServeOptions into a http.Handler that implements
 // all of the given options, in order.
-func makeHandler(n *core.IpfsNode, options ...ServeOption) (http.Handler, error) {
+func MakeHandler(n *core.IpfsNode, options ...ServeOption) (http.Handler, error) {
  topMux := http.NewServeMux()
  mux := topMux
  for _, option := range options {
@@ -49,7 +49,7 @@ func ListenAndServe(n *core.IpfsNode, listeningMultiAddr string, options ...Serv
  if err != nil {
  return err
  }
- handler, err := makeHandler(n, options...)
+ handler, err := MakeHandler(n, options...)
  if err != nil {
  return err
  }

core/corehttp/gateway_test.go

@@ -1,4 +1,4 @@
-package corehttp
+package corehttp_test
 
 import (
  "errors"
@@ -10,12 +10,13 @@ import (
 
  context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
  core "github.com/ipfs/go-ipfs/core"
- coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+ corehttp "github.com/ipfs/go-ipfs/core/corehttp"
  namesys "github.com/ipfs/go-ipfs/namesys"
  ci "github.com/ipfs/go-ipfs/p2p/crypto"
  path "github.com/ipfs/go-ipfs/path"
  repo "github.com/ipfs/go-ipfs/repo"
  config "github.com/ipfs/go-ipfs/repo/config"
+ unixfs "github.com/ipfs/go-ipfs/shell/unixfs"
  testutil "github.com/ipfs/go-ipfs/util/testutil"
 )
 
@@ -60,15 +61,20 @@ func TestGatewayGet(t *testing.T) {
  t.Skip("not sure whats going on here")
  ns := mockNamesys{}
  n := newNodeWithMockNamesys(t, ns)
- k, err := coreunix.Add(n, strings.NewReader("fnord"))
+ dagNode, err := unixfs.AddFromReader(n, strings.NewReader("fnord"))
  if err != nil {
  t.Fatal(err)
  }
+ key, err := dagNode.Key()
+ if err != nil {
+ t.Fatal(err)
+ }
+ k := key.String()
  ns["example.com"] = path.FromString("/ipfs/" + k)
 
- h, err := makeHandler(n,
- IPNSHostnameOption(),
- GatewayOption(false),
+ h, err := corehttp.MakeHandler(n,
+ corehttp.IPNSHostnameOption(),
+ corehttp.GatewayOption(false),
  )
  if err != nil {
  t.Fatal(err)

core/coreunix/add.go

@@ -22,8 +22,12 @@ import (
 
 var log = eventlog.Logger("coreunix")
 
-// Add builds a merkledag from the a reader, pinning all objects to the local
-// datastore. Returns a key representing the root node.
+// DEPRECATED: Add builds a merkledag from the a reader, pinning all
+// objects to the local datastore. Returns a key representing the root
+// node.
+//
+// This function is deprecated and will be removed in 0.4.0. You
+// should use shell/unixfs's AddFromReader instead.
 func Add(n *core.IpfsNode, r io.Reader) (string, error) {
  // TODO more attractive function signature importer.BuildDagFromReader
  dagNode, err := importer.BuildDagFromReader(

docs/release-notes/unreleased.md

@@ -0,0 +1,26 @@
+# `core/coreunix`'s `Add` → `shell/unixfs`'s `AddFromReader`
+
+We've added an `AddFromReader` function to `shell/unixfs`. The old
+`Add` from `core/coreunix` is deprecated and will be removed in
+version 0.4.0. To update your existing code, change usage like:
+
+ keyString, err := coreunix.Add(ipfsNode, reader)
+ if err != nil {
+ return err
+ }
+
+to
+
+ fileNode, err := unixfs.AddFromReader(ipfsNode, reader)
+ if err != nil {
+ return err
+ }
+ key, err := fileNode.Key()
+ if err != nil {
+ return err
+ }
+ keyString := key.String()
+
+That's a bit more verbose if all you want is the stringified key, but
+returning a `dag.Node` makes it easier to perform other operations
+like adding the file node to a directory node.

shell/shell.go

@@ -0,0 +1,8 @@
+/*
+Package shell implements a high-level interface for common IPFS activity.
+
+These wrappers around the low-level core interface make it easy to
+accomplish common tasks with default settings. For steps where the
+defaults aren't appropriate, you can use the core package directly.
+*/
+package shell

shell/unixfs/add.go

@@ -0,0 +1,28 @@
+package unixfs
+
+import (
+ "io"
+
+ core "github.com/ipfs/go-ipfs/core"
+ importer "github.com/ipfs/go-ipfs/importer"
+ chunk "github.com/ipfs/go-ipfs/importer/chunk"
+ dag "github.com/ipfs/go-ipfs/merkledag"
+)
+
+// Add builds a merkledag from a reader, pinning all objects to the
+// local datastore. Returns the root node.
+func AddFromReader(node *core.IpfsNode, reader io.Reader) (*dag.Node, error) {
+ fileNode, err := importer.BuildDagFromReader(
+ reader,
+ node.DAG,
+ node.Pinning.GetManual(),
+ chunk.DefaultSplitter,
+ )
+ if err != nil {
+ return nil, err
+ }
+ if err := node.Pinning.Flush(); err != nil {
+ return nil, err
+ }
+ return fileNode, nil
+}

shell/unixfs/unixfs.go

@@ -0,0 +1,10 @@
+/*
+Package unixfs is a high-level interface for filesystem nodes.
+
+Simple wrappers to:
+
+* create file and directory nodes from paths and readers,
+* extract file and directory nodes to your local filesytem, and
+* print file contents to writers.
+*/
+package unixfs

test/integration/addcat_test.go

@@ -4,7 +4,6 @@ import (
  "bytes"
  "errors"
  "fmt"
- "io"
  "math"
  "os"
  "testing"
@@ -14,11 +13,11 @@ import (
  context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
  "github.com/ipfs/go-ipfs/core"
- coreunix "github.com/ipfs/go-ipfs/core/coreunix"
  mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
  "github.com/ipfs/go-ipfs/p2p/peer"
  "github.com/ipfs/go-ipfs/thirdparty/unit"
  testutil "github.com/ipfs/go-ipfs/util/testutil"
+ addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 const kSeed = 1
@@ -126,22 +125,10 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
  return err
  }
 
- added, err := coreunix.Add(adder, bytes.NewReader(data))
+ err = addcat.AddCat(adder, catter, data)
  if err != nil {
  return err
  }
-
- readerCatted, err := coreunix.Cat(catter, added)
- if err != nil {
- return err
- }
-
- // verify
- var bufout bytes.Buffer
- io.Copy(&bufout, readerCatted)
- if 0 != bytes.Compare(bufout.Bytes(), data) {
- return errors.New("catted data does not match added data")
- }
  return nil
 }
 

test/integration/bench_cat_test.go

@@ -1,19 +1,17 @@
 package integrationtest
 
 import (
- "bytes"
  "errors"
- "io"
  "math"
  "testing"
 
  context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
  "github.com/ipfs/go-ipfs/core"
- coreunix "github.com/ipfs/go-ipfs/core/coreunix"
  mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
  "github.com/ipfs/go-ipfs/p2p/peer"
  "github.com/ipfs/go-ipfs/thirdparty/unit"
  testutil "github.com/ipfs/go-ipfs/util/testutil"
+ addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func BenchmarkCat1MB(b *testing.B) { benchmarkVarCat(b, unit.MB*1) }
@@ -74,22 +72,9 @@ func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
  return err
  }
 
- added, err := coreunix.Add(adder, bytes.NewReader(data))
+ err = addcat.AddCat(adder, catter, data)
  if err != nil {
  return err
  }
-
- b.StartTimer()
- readerCatted, err := coreunix.Cat(catter, added)
- if err != nil {
- return err
- }
-
- // verify
- var bufout bytes.Buffer
- io.Copy(&bufout, readerCatted)
- if 0 != bytes.Compare(bufout.Bytes(), data) {
- return errors.New("catted data does not match added data")
- }
  return nil
 }

test/integration/grandcentral_test.go

@@ -4,7 +4,6 @@ import (
  "bytes"
  "errors"
  "fmt"
- "io"
  "math"
  "testing"
 
@@ -14,14 +13,14 @@ import (
 
  core "github.com/ipfs/go-ipfs/core"
  "github.com/ipfs/go-ipfs/core/corerouting"
- "github.com/ipfs/go-ipfs/core/coreunix"
  mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
  "github.com/ipfs/go-ipfs/p2p/peer"
  "github.com/ipfs/go-ipfs/thirdparty/iter"
  "github.com/ipfs/go-ipfs/thirdparty/unit"
  "github.com/ipfs/go-ipfs/util"
  ds2 "github.com/ipfs/go-ipfs/util/datastore2"
  testutil "github.com/ipfs/go-ipfs/util/testutil"
+ addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func TestSupernodeBootstrappedAddCat(t *testing.T) {
@@ -54,25 +53,10 @@ func RunSupernodeBootstrappedAddCat(data []byte, conf testutil.LatencyConfig) er
  adder := clients[0]
  catter := clients[1]
 
- log.Info("adder is", adder.Identity)
- log.Info("catter is", catter.Identity)
-
- keyAdded, err := coreunix.Add(adder, bytes.NewReader(data))
+ err = addcat.AddCat(adder, catter, data)
  if err != nil {
  return err
  }
-
- readerCatted, err := coreunix.Cat(catter, keyAdded)
- if err != nil {
- return err
- }
-
- // verify
- var bufout bytes.Buffer
- io.Copy(&bufout, readerCatted)
- if 0 != bytes.Compare(bufout.Bytes(), data) {
- return errors.New("catted data does not match added data")
- }
  return nil
 }