From 8cd3b569fb7e69aedeca985ab24ab7957fe1400c Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 26 Sep 2019 13:01:18 +0200 Subject: [PATCH 1/4] pushsync: optimize TestPushsyncSimulation --- chunk/testing/chunk.go | 54 +++++++++++++++++++++++++++ pushsync/simulation_test.go | 10 +++-- storage/localstore/localstore_test.go | 34 +++-------------- 3 files changed, 66 insertions(+), 32 deletions(-) create mode 100644 chunk/testing/chunk.go diff --git a/chunk/testing/chunk.go b/chunk/testing/chunk.go new file mode 100644 index 0000000000..fa8b749d5b --- /dev/null +++ b/chunk/testing/chunk.go @@ -0,0 +1,54 @@ +// Copyright 2019 The Swarm Authors +// This file is part of the Swarm library. +// +// The Swarm library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Swarm library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Swarm library. If not, see . + +package testing + +import ( + "math/rand" + "time" + + "github.com/ethersphere/swarm/chunk" +) + +func init() { + // needed for GenerateTestRandomChunk + rand.Seed(time.Now().UnixNano()) +} + +// GenerateTestRandomChunk generates a Chunk that is not +// valid, but it contains a random key and a random value. +// This function is faster then storage.GenerateRandomChunk +// which generates a valid chunk. +// Some tests in do not need valid chunks, just +// random data, and their execution time can be decreased +// using this function. +func GenerateTestRandomChunk() chunk.Chunk { + data := make([]byte, chunk.DefaultSize) + rand.Read(data) + key := make([]byte, 32) + rand.Read(key) + return chunk.NewChunk(key, data) +} + +// GenerateTestRandomChunks generates a slice of random +// Chunks by using GenerateTestRandomChunk function. +func GenerateTestRandomChunks(count int) []chunk.Chunk { + chunks := make([]chunk.Chunk, count) + for i := 0; i < count; i++ { + chunks[i] = GenerateTestRandomChunk() + } + return chunks +} diff --git a/pushsync/simulation_test.go b/pushsync/simulation_test.go index a2dc131cc0..794152ccb6 100644 --- a/pushsync/simulation_test.go +++ b/pushsync/simulation_test.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/swarm/chunk" + chunktesting "github.com/ethersphere/swarm/chunk/testing" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network" "github.com/ethersphere/swarm/network/retrieval" @@ -52,7 +53,7 @@ var ( var ( nodeCntFlag = flag.Int("nodes", 4, "number of nodes in simulation") - chunkCntFlag = flag.Int("chunks", 4, "number of chunks per upload in simulation") + chunkCntFlag = flag.Int("chunks", 1000, "number of chunks per upload in simulation") testCasesFlag = flag.Int("cases", 4, "number of concurrent upload-download cases to test in simulation") ) @@ -140,7 +141,7 @@ func uploadAndDownload(ctx context.Context, sim *simulation.Simulation, nodeCnt, log.Debug("uploaded", "peer", uid, "chunks", chunkCnt, "tagname", tagname) // wait till pushsync is done - syncTimeout := 30 * time.Second + syncTimeout := 120 * time.Second sctx, cancel := context.WithTimeout(ctx, syncTimeout) defer cancel() err = tag.WaitTillDone(sctx, chunk.StateSynced) @@ -244,7 +245,7 @@ func upload(ctx context.Context, store Store, tags *chunk.Tags, tagname string, return nil, nil, err } for i := 0; i < n; i++ { - ch := storage.GenerateRandomChunk(int64(chunk.DefaultSize)) + ch := chunktesting.GenerateTestRandomChunk() addrs = append(addrs, ch.Address()) _, err := store.Put(ctx, chunk.ModePutUpload, ch.WithTagID(tag.Uid)) if err != nil { @@ -257,9 +258,12 @@ func upload(ctx context.Context, store Store, tags *chunk.Tags, tagname string, func download(ctx context.Context, store *storage.NetStore, addrs []storage.Address) error { var g errgroup.Group + sem := make(chan struct{}, 32) // limit the number of concurrent gets for _, addr := range addrs { addr := addr + sem <- struct{}{} g.Go(func() error { + defer func() { <-sem }() _, err := store.Get(ctx, chunk.ModeGetRequest, storage.NewRequest(addr)) log.Debug("Get", "addr", hex.EncodeToString(addr[:]), "err", err) return err diff --git a/storage/localstore/localstore_test.go b/storage/localstore/localstore_test.go index 173d6ffb73..ccfd0de946 100644 --- a/storage/localstore/localstore_test.go +++ b/storage/localstore/localstore_test.go @@ -30,6 +30,7 @@ import ( "time" "github.com/ethersphere/swarm/chunk" + chunktesting "github.com/ethersphere/swarm/chunk/testing" "github.com/ethersphere/swarm/shed" "github.com/syndtr/goleveldb/leveldb" ) @@ -165,35 +166,10 @@ func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) { return db, cleanupFunc } -func init() { - // needed for generateTestRandomChunk - rand.Seed(time.Now().UnixNano()) -} - -// generateTestRandomChunk generates a Chunk that is not -// valid, but it contains a random key and a random value. -// This function is faster then storage.generateTestRandomChunk -// which generates a valid chunk. -// Some tests in this package do not need valid chunks, just -// random data, and their execution time can be decreased -// using this function. -func generateTestRandomChunk() chunk.Chunk { - data := make([]byte, chunk.DefaultSize) - rand.Read(data) - key := make([]byte, 32) - rand.Read(key) - return chunk.NewChunk(key, data) -} - -// generateTestRandomChunks generates a slice of random -// Chunks by using generateTestRandomChunk function. -func generateTestRandomChunks(count int) []chunk.Chunk { - chunks := make([]chunk.Chunk, count) - for i := 0; i < count; i++ { - chunks[i] = generateTestRandomChunk() - } - return chunks -} +var ( + generateTestRandomChunk = chunktesting.GenerateTestRandomChunk + generateTestRandomChunks = chunktesting.GenerateTestRandomChunks +) // chunkAddresses return chunk addresses of provided chunks. func chunkAddresses(chunks []chunk.Chunk) []chunk.Address { From 0a5d70f78c76a7bdc714313963956ae29136cbd5 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 26 Sep 2019 14:14:34 +0200 Subject: [PATCH 2/4] pushsync: reduce the number of test cases for TestPushsyncSimulation --- pushsync/simulation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pushsync/simulation_test.go b/pushsync/simulation_test.go index 794152ccb6..3c030bd55c 100644 --- a/pushsync/simulation_test.go +++ b/pushsync/simulation_test.go @@ -54,7 +54,7 @@ var ( var ( nodeCntFlag = flag.Int("nodes", 4, "number of nodes in simulation") chunkCntFlag = flag.Int("chunks", 1000, "number of chunks per upload in simulation") - testCasesFlag = flag.Int("cases", 4, "number of concurrent upload-download cases to test in simulation") + testCasesFlag = flag.Int("cases", 2, "number of concurrent upload-download cases to test in simulation") ) // test syncer using pss From 9f95bd60e6e3422758443d4d699fb4d67eb1ff34 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 26 Sep 2019 15:25:52 +0200 Subject: [PATCH 3/4] pushsync: adjust TestPushsyncSimulation defaults for travis --- pushsync/simulation_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pushsync/simulation_test.go b/pushsync/simulation_test.go index 3c030bd55c..440a3a992c 100644 --- a/pushsync/simulation_test.go +++ b/pushsync/simulation_test.go @@ -53,8 +53,8 @@ var ( var ( nodeCntFlag = flag.Int("nodes", 4, "number of nodes in simulation") - chunkCntFlag = flag.Int("chunks", 1000, "number of chunks per upload in simulation") - testCasesFlag = flag.Int("cases", 2, "number of concurrent upload-download cases to test in simulation") + chunkCntFlag = flag.Int("chunks", 100, "number of chunks per upload in simulation") + testCasesFlag = flag.Int("cases", 4, "number of concurrent upload-download cases to test in simulation") ) // test syncer using pss From 8ae73d7e3b4c4895d3664052d2bd8cd05f152157 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 26 Sep 2019 17:19:11 +0200 Subject: [PATCH 4/4] piushsync: reduce default simulation chunk count on appveyor --- pushsync/simulation_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pushsync/simulation_test.go b/pushsync/simulation_test.go index 440a3a992c..03a20a63f7 100644 --- a/pushsync/simulation_test.go +++ b/pushsync/simulation_test.go @@ -52,11 +52,19 @@ var ( ) var ( + defaultChunkCnt = 100 + nodeCntFlag = flag.Int("nodes", 4, "number of nodes in simulation") - chunkCntFlag = flag.Int("chunks", 100, "number of chunks per upload in simulation") + chunkCntFlag = flag.Int("chunks", defaultChunkCnt, "number of chunks per upload in simulation") testCasesFlag = flag.Int("cases", 4, "number of concurrent upload-download cases to test in simulation") ) +func init() { + if os.Getenv("APPVEYOR") != "" { + defaultChunkCnt = 4 + } +} + // test syncer using pss // the test // * creates a simulation with connectivity loaded from a snapshot