util/testutil/addcat/addcat: Factor out add/cat/verify helper

Save some repetitive logic in the integration tests. While we're at it, replace coreunix.Add with a directo call to BuildDagFromReader. coreunix.Add is basically a wrapper around BuildDagFromReader that returns a string-ified key instead of a DAG node. That's what we want here, so it's a few more lines to inline the key extraction (mostly due to Go's explicit error handling [1], which makes method chaining not idomatic [2]). But I think a bit of local error-checking is a reasonable price for a more concise API, where there's only one function for adding file nodes from a reader. I'd planned to add the AddCat helper to the testutil package, but that triggered some cyclic imports: $ make test cd cmd/ipfs && go build -i import cycle not allowed package github.com/ipfs/go-ipfs/cmd/ipfs imports github.com/ipfs/go-ipfs/commands imports github.com/ipfs/go-ipfs/core imports github.com/ipfs/go-ipfs/blockservice imports github.com/ipfs/go-ipfs/exchange/bitswap imports github.com/ipfs/go-ipfs/exchange/bitswap/testnet imports github.com/ipfs/go-ipfs/p2p/net/mock imports github.com/ipfs/go-ipfs/p2p/test/util imports github.com/ipfs/go-ipfs/util/testutil imports github.com/ipfs/go-ipfs/core/coreunix imports github.com/ipfs/go-ipfs/importer imports github.com/ipfs/go-ipfs/importer/balanced imports github.com/ipfs/go-ipfs/importer/helpers imports github.com/ipfs/go-ipfs/merkledag imports github.com/ipfs/go-ipfs/blockservice import cycle not allowed package github.com/ipfs/go-ipfs/cmd/ipfs imports github.com/ipfs/go-ipfs/commands imports github.com/ipfs/go-ipfs/core imports github.com/ipfs/go-ipfs/blockservice imports github.com/ipfs/go-ipfs/exchange/bitswap imports github.com/ipfs/go-ipfs/exchange/bitswap/testnet imports github.com/ipfs/go-ipfs/p2p/net/mock imports github.com/ipfs/go-ipfs/p2p/test/util imports github.com/ipfs/go-ipfs/util/testutil imports github.com/ipfs/go-ipfs/core Makefile:27: recipe for target 'build' failed make: *** [build] Error 1 I think the proper workaround is to split tests that import test-only libraries out into their own packages. For example, exchange/bitswap/bitswap_test.go is in the bitswap package, but imports the exchange/bitswap/testnet package, which in turn pulls in mock, testutil, etc. The easiest workaround seems to be using external tests [3] (e.g. bitswap_test for exchange/bitswap/bitswap_test.go). I tried to apply this approach, but the current codebase had too many crosslinks (e.g. internal tests that import test-only packages but also use internal utilities from their current package), so I gave up. I think gradually converting test files into external tests is a good thing, but it's too much to bite off for this feature branch. [1]: https://golang.org/doc/faq#exceptions [2]: http://stackoverflow.com/a/27300387 [3]: http://dave.cheney.net/2014/12/01/five-suggestions-for-setting-up-a-go-project
ipfs · Apr 27, 2015 · 5a076f0 · 5a076f0
1 parent 068adad
commit 5a076f0
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 81 deletions.
diff --git a/test/integration/addcat_test.go b/test/integration/addcat_test.go
@@ -3,7 +3,6 @@ package integrationtest
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"math"
 	"os"
 	"testing"
@@ -12,12 +11,12 @@ import (
 	random "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random"
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	errors "github.com/ipfs/go-ipfs/util/debugerror"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 const kSeed = 1
@@ -29,7 +28,7 @@ func Test1KBInstantaneous(t *testing.T) {
 		BlockstoreLatency: 0,
 	}
 
-	if err := DirectAddCat(RandomBytes(1*unit.KB), conf); err != nil {
+	if err := DirectAddCat(string(RandomBytes(1*unit.KB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -61,7 +60,7 @@ func TestDegenerateSlowRouting(t *testing.T) {
 func Test100MBMacbookCoastToCoast(t *testing.T) {
 	SkipUnlessEpic(t)
 	conf := testutil.LatencyConfig{}.Network_NYtoSF().Blockstore_SlowSSD2014().Routing_Slow()
-	if err := DirectAddCat(RandomBytes(100*1024*1024), conf); err != nil {
+	if err := DirectAddCat(string(RandomBytes(100*1024*1024)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -70,7 +69,7 @@ func AddCatPowers(conf testutil.LatencyConfig, megabytesMax int64) error {
 	var i int64
 	for i = 1; i < megabytesMax; i = i * 2 {
 		fmt.Printf("%d MB\n", i)
-		if err := DirectAddCat(RandomBytes(i*1024*1024), conf); err != nil {
+		if err := DirectAddCat(string(RandomBytes(i*1024*1024)), conf); err != nil {
 			return err
 		}
 	}
@@ -83,7 +82,7 @@ func RandomBytes(n int64) []byte {
 	return data.Bytes()
 }
 
-func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
+func DirectAddCat(data string, conf testutil.LatencyConfig) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	const numPeers = 2
@@ -125,22 +124,10 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
 		return err
 	}
 
-	added, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, string(data))
 	if err != nil {
 		return err
 	}
-
-	readerCatted, err := coreunix.Cat(catter, added)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }
 

diff --git a/test/integration/bench_cat_test.go b/test/integration/bench_cat_test.go
@@ -1,27 +1,25 @@
 package integrationtest
 
 import (
-	"bytes"
-	"io"
 	"math"
 	"testing"
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	errors "github.com/ipfs/go-ipfs/util/debugerror"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func BenchmarkCat1MB(b *testing.B) { benchmarkVarCat(b, unit.MB*1) }
 func BenchmarkCat2MB(b *testing.B) { benchmarkVarCat(b, unit.MB*2) }
 func BenchmarkCat4MB(b *testing.B) { benchmarkVarCat(b, unit.MB*4) }
 
 func benchmarkVarCat(b *testing.B, size int64) {
-	data := RandomBytes(size)
+	data := string(RandomBytes(size))
 	b.SetBytes(size)
 	for n := 0; n < b.N; n++ {
 		err := benchCat(b, data, instant)
@@ -31,7 +29,7 @@ func benchmarkVarCat(b *testing.B, size int64) {
 	}
 }
 
-func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
+func benchCat(b *testing.B, data string, conf testutil.LatencyConfig) error {
 	b.StopTimer()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -74,22 +72,9 @@ func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
 		return err
 	}
 
-	added, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, data)
 	if err != nil {
 		return err
 	}
-
-	b.StartTimer()
-	readerCatted, err := coreunix.Cat(catter, added)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }
diff --git a/test/integration/bench_test.go b/test/integration/bench_test.go
@@ -11,7 +11,7 @@ func benchmarkAddCat(numBytes int64, conf testutil.LatencyConfig, b *testing.B)
 
 	b.StopTimer()
 	b.SetBytes(numBytes)
-	data := RandomBytes(numBytes) // we don't want to measure the time it takes to generate this data
+	data := string(RandomBytes(numBytes)) // we don't want to measure the time it takes to generate this data
 	b.StartTimer()
 
 	for n := 0; n < b.N; n++ {

diff --git a/test/integration/grandcentral_test.go b/test/integration/grandcentral_test.go
@@ -3,7 +3,6 @@ package integrationtest
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"math"
 	"testing"
 
@@ -12,7 +11,6 @@ import (
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	core "github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/core/corerouting"
-	"github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/iter"
@@ -21,6 +19,7 @@ import (
 	ds2 "github.com/ipfs/go-ipfs/util/datastore2"
 	errors "github.com/ipfs/go-ipfs/util/debugerror"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func TestSupernodeBootstrappedAddCat(t *testing.T) {
@@ -33,12 +32,12 @@ func TestSupernodeBootstrappedAddCat(t *testing.T) {
 		RoutingLatency:    0,
 		BlockstoreLatency: 0,
 	}
-	if err := RunSupernodeBootstrappedAddCat(RandomBytes(100*unit.MB), conf); err != nil {
+	if err := RunSupernodeBootstrappedAddCat(string(RandomBytes(100*unit.MB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
 
-func RunSupernodeBootstrappedAddCat(data []byte, conf testutil.LatencyConfig) error {
+func RunSupernodeBootstrappedAddCat(data string, conf testutil.LatencyConfig) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -53,25 +52,10 @@ func RunSupernodeBootstrappedAddCat(data []byte, conf testutil.LatencyConfig) er
 	adder := clients[0]
 	catter := clients[1]
 
-	log.Info("adder is", adder.Identity)
-	log.Info("catter is", catter.Identity)
-
-	keyAdded, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, data)
 	if err != nil {
 		return err
 	}
-
-	readerCatted, err := coreunix.Cat(catter, keyAdded)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }
 

diff --git a/test/integration/three_legged_cat_test.go b/test/integration/three_legged_cat_test.go
@@ -1,20 +1,18 @@
 package integrationtest
 
 import (
-	"bytes"
-	"io"
 	"math"
 	"testing"
 	"time"
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	core "github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	errors "github.com/ipfs/go-ipfs/util/debugerror"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func TestThreeLeggedCatTransfer(t *testing.T) {
@@ -23,44 +21,44 @@ func TestThreeLeggedCatTransfer(t *testing.T) {
 		RoutingLatency:    0,
 		BlockstoreLatency: 0,
 	}
-	if err := RunThreeLeggedCat(RandomBytes(100*unit.MB), conf); err != nil {
+	if err := RunThreeLeggedCat(string(RandomBytes(100*unit.MB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestThreeLeggedCatDegenerateSlowBlockstore(t *testing.T) {
 	SkipUnlessEpic(t)
 	conf := testutil.LatencyConfig{BlockstoreLatency: 50 * time.Millisecond}
-	if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil {
+	if err := RunThreeLeggedCat(string(RandomBytes(1*unit.KB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestThreeLeggedCatDegenerateSlowNetwork(t *testing.T) {
 	SkipUnlessEpic(t)
 	conf := testutil.LatencyConfig{NetworkLatency: 400 * time.Millisecond}
-	if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil {
+	if err := RunThreeLeggedCat(string(RandomBytes(1*unit.KB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestThreeLeggedCatDegenerateSlowRouting(t *testing.T) {
 	SkipUnlessEpic(t)
 	conf := testutil.LatencyConfig{RoutingLatency: 400 * time.Millisecond}
-	if err := RunThreeLeggedCat(RandomBytes(1*unit.KB), conf); err != nil {
+	if err := RunThreeLeggedCat(string(RandomBytes(1*unit.KB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestThreeLeggedCat100MBMacbookCoastToCoast(t *testing.T) {
 	SkipUnlessEpic(t)
 	conf := testutil.LatencyConfig{}.Network_NYtoSF().Blockstore_SlowSSD2014().Routing_Slow()
-	if err := RunThreeLeggedCat(RandomBytes(100*unit.MB), conf); err != nil {
+	if err := RunThreeLeggedCat(string(RandomBytes(100*unit.MB)), conf); err != nil {
 		t.Fatal(err)
 	}
 }
 
-func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
+func RunThreeLeggedCat(data string, conf testutil.LatencyConfig) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	const numPeers = 3
@@ -105,21 +103,9 @@ func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
 		return err
 	}
 
-	added, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, data)
 	if err != nil {
 		return err
 	}
-
-	readerCatted, err := coreunix.Cat(catter, added)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }
diff --git a/util/testutil/addcat/addcat.go b/util/testutil/addcat/addcat.go
@@ -0,0 +1,44 @@
+package testutil
+
+import (
+	"bytes"
+	"strings"
+
+	core "github.com/ipfs/go-ipfs/core"
+	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+	importer "github.com/ipfs/go-ipfs/importer"
+	chunk "github.com/ipfs/go-ipfs/importer/chunk"
+	errors "github.com/ipfs/go-ipfs/util/debugerror"
+)
+
+func AddCat(adder *core.IpfsNode, catter *core.IpfsNode, data string) error {
+	dagNode, err := importer.BuildDagFromReader(
+		strings.NewReader(data),
+		adder.DAG,
+		adder.Pinning.GetManual(),
+		chunk.DefaultSplitter)
+	if err != nil {
+		return err
+	}
+	key, err := dagNode.Key()
+	if err != nil {
+		return err
+	}
+	added := key.String()
+
+	reader, err := coreunix.Cat(catter, added)
+	if err != nil {
+		return err
+	}
+
+	// verify
+	var buf bytes.Buffer
+	_, err = buf.ReadFrom(reader)
+	if err != nil {
+		return err
+	}
+	if buf.String() != data {
+		return errors.New("catted data does not match added data")
+	}
+	return nil
+}