pushsync: optimize TestPushsyncSimulation

chunk/testing/chunk.go

@@ -0,0 +1,54 @@
+// Copyright 2019 The Swarm Authors
+// This file is part of the Swarm library.
+//
+// The Swarm library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The Swarm library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the Swarm library. If not, see <http://www.gnu.org/licenses/>.
+
+package testing
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/ethersphere/swarm/chunk"
+)
+
+func init() {
+	// needed for GenerateTestRandomChunk
+	rand.Seed(time.Now().UnixNano())
+}
+
+// GenerateTestRandomChunk generates a Chunk that is not
+// valid, but it contains a random key and a random value.
+// This function is faster then storage.GenerateRandomChunk
+// which generates a valid chunk.
+// Some tests in do not need valid chunks, just
+// random data, and their execution time can be decreased
+// using this function.
+func GenerateTestRandomChunk() chunk.Chunk {
+	data := make([]byte, chunk.DefaultSize)
+	rand.Read(data)
+	key := make([]byte, 32)
+	rand.Read(key)
+	return chunk.NewChunk(key, data)
+}
+
+// GenerateTestRandomChunks generates a slice of random
+// Chunks by using GenerateTestRandomChunk function.
+func GenerateTestRandomChunks(count int) []chunk.Chunk {
+	chunks := make([]chunk.Chunk, count)
+	for i := 0; i < count; i++ {
+		chunks[i] = GenerateTestRandomChunk()
+	}
+	return chunks
+}

pushsync/simulation_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethersphere/swarm/chunk"
+	chunktesting "github.com/ethersphere/swarm/chunk/testing"
 	"github.com/ethersphere/swarm/log"
 	"github.com/ethersphere/swarm/network"
 	"github.com/ethersphere/swarm/network/retrieval"
@@ -51,11 +52,19 @@ var (
 )
 
 var (
+	defaultChunkCnt = 100
+
 	nodeCntFlag   = flag.Int("nodes", 4, "number of nodes in simulation")
-	chunkCntFlag  = flag.Int("chunks", 4, "number of chunks per upload in simulation")
+	chunkCntFlag  = flag.Int("chunks", defaultChunkCnt, "number of chunks per upload in simulation")
 	testCasesFlag = flag.Int("cases", 4, "number of concurrent upload-download cases to test in simulation")
 )
 
+func init() {
+	if os.Getenv("APPVEYOR") != "" {
+		defaultChunkCnt = 4
+	}
+}
+
 // test syncer using pss
 // the test
 // * creates a simulation with connectivity loaded from a snapshot
@@ -140,7 +149,7 @@ func uploadAndDownload(ctx context.Context, sim *simulation.Simulation, nodeCnt,
 	log.Debug("uploaded", "peer", uid, "chunks", chunkCnt, "tagname", tagname)
 
 	// wait till pushsync is done
-	syncTimeout := 30 * time.Second
+	syncTimeout := 120 * time.Second
 	sctx, cancel := context.WithTimeout(ctx, syncTimeout)
 	defer cancel()
 	err = tag.WaitTillDone(sctx, chunk.StateSynced)
@@ -244,7 +253,7 @@ func upload(ctx context.Context, store Store, tags *chunk.Tags, tagname string,
 		return nil, nil, err
 	}
 	for i := 0; i < n; i++ {
-		ch := storage.GenerateRandomChunk(int64(chunk.DefaultSize))
+		ch := chunktesting.GenerateTestRandomChunk()
 		addrs = append(addrs, ch.Address())
 		_, err := store.Put(ctx, chunk.ModePutUpload, ch.WithTagID(tag.Uid))
 		if err != nil {
@@ -257,9 +266,12 @@ func upload(ctx context.Context, store Store, tags *chunk.Tags, tagname string,
 
 func download(ctx context.Context, store *storage.NetStore, addrs []storage.Address) error {
 	var g errgroup.Group
+	sem := make(chan struct{}, 32) // limit the number of concurrent gets
 	for _, addr := range addrs {
 		addr := addr
+		sem <- struct{}{}
 		g.Go(func() error {
+			defer func() { <-sem }()
 			_, err := store.Get(ctx, chunk.ModeGetRequest, storage.NewRequest(addr))
 			log.Debug("Get", "addr", hex.EncodeToString(addr[:]), "err", err)
 			return err

storage/localstore/localstore_test.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/ethersphere/swarm/chunk"
+	chunktesting "github.com/ethersphere/swarm/chunk/testing"
 	"github.com/ethersphere/swarm/shed"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -165,35 +166,10 @@ func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) {
 	return db, cleanupFunc
 }
 
-func init() {
-	// needed for generateTestRandomChunk
-	rand.Seed(time.Now().UnixNano())
-}
-
-// generateTestRandomChunk generates a Chunk that is not
-// valid, but it contains a random key and a random value.
-// This function is faster then storage.generateTestRandomChunk
-// which generates a valid chunk.
-// Some tests in this package do not need valid chunks, just
-// random data, and their execution time can be decreased
-// using this function.
-func generateTestRandomChunk() chunk.Chunk {
-	data := make([]byte, chunk.DefaultSize)
-	rand.Read(data)
-	key := make([]byte, 32)
-	rand.Read(key)
-	return chunk.NewChunk(key, data)
-}
-
-// generateTestRandomChunks generates a slice of random
-// Chunks by using generateTestRandomChunk function.
-func generateTestRandomChunks(count int) []chunk.Chunk {
-	chunks := make([]chunk.Chunk, count)
-	for i := 0; i < count; i++ {
-		chunks[i] = generateTestRandomChunk()
-	}
-	return chunks
-}
+var (
+	generateTestRandomChunk  = chunktesting.GenerateTestRandomChunk
+	generateTestRandomChunks = chunktesting.GenerateTestRandomChunks
+)
 
 // chunkAddresses return chunk addresses of provided chunks.
 func chunkAddresses(chunks []chunk.Chunk) []chunk.Address {