diff --git a/benchmarks_test.go b/benchmarks_test.go index 4293a987..01439d7c 100644 --- a/benchmarks_test.go +++ b/benchmarks_test.go @@ -13,6 +13,7 @@ import ( "github.com/ipfs/go-bitswap/testutil" blocks "github.com/ipfs/go-block-format" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" bitswap "github.com/ipfs/go-bitswap" bssession "github.com/ipfs/go-bitswap/session" @@ -24,6 +25,8 @@ import ( mockrouting "github.com/ipfs/go-ipfs-routing/mock" ) +type topologyFunc func(b *testing.B, instances []testinstance.Instance) + type fetchFunc func(b *testing.B, bs *bitswap.Bitswap, ks []cid.Cid) type distFunc func(b *testing.B, provs []testinstance.Instance, blocks []blocks.Block) @@ -152,12 +155,49 @@ func BenchmarkDupsManyNodesRealWorldNetwork(b *testing.B) { ioutil.WriteFile("tmp/rw-benchmark.json", out, 0666) } +func BenchmarkDupsManyNodesRealWorldNetworkWithRealDHT(b *testing.B) { + benchmarkLog = nil + benchmarkSeed, err := strconv.ParseInt(os.Getenv("BENCHMARK_SEED"), 10, 64) + var randomGen *rand.Rand = nil + if err == nil { + randomGen = rand.New(rand.NewSource(benchmarkSeed)) + } + + fastNetworkDelayGenerator := tn.InternetLatencyDelayGenerator( + mediumSpeed-fastSpeed, slowSpeed-fastSpeed, + 0.0, 0.0, distribution, randomGen) + fastNetworkDelay := delay.Delay(fastSpeed, fastNetworkDelayGenerator) + fastBandwidthGenerator := tn.VariableRateLimitGenerator(fastBandwidth, fastBandwidthDeviation, randomGen) + averageNetworkDelayGenerator := tn.InternetLatencyDelayGenerator( + mediumSpeed-fastSpeed, slowSpeed-fastSpeed, + 0.3, 0.3, distribution, randomGen) + averageNetworkDelay := delay.Delay(fastSpeed, averageNetworkDelayGenerator) + averageBandwidthGenerator := tn.VariableRateLimitGenerator(mediumBandwidth, mediumBandwidthDeviation, randomGen) + slowNetworkDelayGenerator := tn.InternetLatencyDelayGenerator( + mediumSpeed-fastSpeed, superSlowSpeed-fastSpeed, + 0.3, 0.3, distribution, randomGen) + slowNetworkDelay := delay.Delay(fastSpeed, slowNetworkDelayGenerator) + slowBandwidthGenerator := tn.VariableRateLimitGenerator(slowBandwidth, slowBandwidthDeviation, randomGen) + + b.Run("10Nodes-RandomRandom-BigBatch-FastNetwork", func(b *testing.B) { + subtestDistributeAndFetchWithRealDHT(b, 20, 10, fastNetworkDelay, fastBandwidthGenerator, stdBlockSize, starPattern, randomRandom, batchFetchAll) + }) + b.Run("10Nodes-RandomRandom-BigBatch-AverageVariableSpeedNetwork", func(b *testing.B) { + subtestDistributeAndFetchWithRealDHT(b, 20, 10, averageNetworkDelay, averageBandwidthGenerator, stdBlockSize, starPattern, randomRandom, batchFetchAll) + }) + b.Run("10Nodes-RandomRandom-BigBatch-SlowVariableSpeedNetwork", func(b *testing.B) { + subtestDistributeAndFetchWithRealDHT(b, 20, 10, slowNetworkDelay, slowBandwidthGenerator, stdBlockSize, starPattern, randomRandom, batchFetchAll) + }) + out, _ := json.MarshalIndent(benchmarkLog, "", " ") + ioutil.WriteFile("tmp/rw-benchmark.json", out, 0666) +} + func subtestDistributeAndFetch(b *testing.B, numnodes, numblks int, d delay.D, df distFunc, ff fetchFunc) { for i := 0; i < b.N; i++ { start := time.Now() net := tn.VirtualNetwork(mockrouting.NewServer(), d) - ig := testinstance.NewTestInstanceGenerator(net) + ig := testinstance.NewTestInstanceGenerator(net, bitswap.ProvideEnabled(false)) defer ig.Close() bg := blocksutil.NewBlockGenerator() @@ -174,7 +214,7 @@ func subtestDistributeAndFetchRateLimited(b *testing.B, numnodes, numblks int, d start := time.Now() net := tn.RateLimitedVirtualNetwork(mockrouting.NewServer(), d, rateLimitGenerator) - ig := testinstance.NewTestInstanceGenerator(net) + ig := testinstance.NewTestInstanceGenerator(net, bitswap.ProvideEnabled(false)) defer ig.Close() instances := ig.Instances(numnodes) @@ -184,6 +224,45 @@ func subtestDistributeAndFetchRateLimited(b *testing.B, numnodes, numblks int, d } } +func subtestDistributeAndFetchWithRealDHT(b *testing.B, numnodes, numblks int, d delay.D, rateLimitGenerator tn.RateLimitGenerator, blockSize int64, tf topologyFunc, df distFunc, ff fetchFunc) { + for i := 0; i < b.N; i++ { + start := time.Now() + ctx := context.Background() + mn := mocknet.New(ctx) + routing := tn.NewDHTServer() + net, err := tn.StreamNet(ctx, mn, routing) + if err != nil { + b.Fatal("Unable to initialize mocknet") + } + + ig := testinstance.NewTestInstanceGenerator(net) + defer ig.Close() + + var instances []testinstance.Instance + for j := 0; j < numnodes; j++ { + inst := ig.Next() + instances = append(instances, inst) + } + mn.LinkAll() + for i, inst := range instances { + // rate limit and delate connections + for j := i + 1; j < len(instances); j++ { + oinst := instances[j] + links := mn.LinksBetweenPeers(inst.Peer, oinst.Peer) + for _, link := range links { + link.SetOptions(mocknet.LinkOptions{Latency: d.NextWaitTime(), Bandwidth: rateLimitGenerator.NextRateLimit()}) + } + } + } + + tf(b, instances) + + blocks := testutil.GenerateBlocksOfSize(numblks, blockSize) + + runDistribution(b, instances, blocks, df, ff, start) + } +} + func runDistribution(b *testing.B, instances []testinstance.Instance, blocks []blocks.Block, df distFunc, ff fetchFunc, start time.Time) { numnodes := len(instances) @@ -216,13 +295,25 @@ func runDistribution(b *testing.B, instances []testinstance.Instance, blocks []b b.Logf("send/recv: %d / %d", nst.MessagesSent, nst.MessagesRecvd) } -func allToAll(b *testing.B, provs []testinstance.Instance, blocks []blocks.Block) { - for _, p := range provs { - if err := p.Blockstore().PutMany(blocks); err != nil { +func starPattern(b *testing.B, instances []testinstance.Instance) { + for _, inst := range instances { + // Connect in a star pattern to first node only (like a bootstrap) + inst.Adapter.ConnectTo(context.Background(), instances[0].Peer) + } +} + +func putMany(b *testing.B, p testinstance.Instance, blocks []blocks.Block) { + for _, blk := range blocks { + if err := p.Exchange.HasBlock(blk); err != nil { b.Fatal(err) } } } +func allToAll(b *testing.B, provs []testinstance.Instance, blocks []blocks.Block) { + for _, p := range provs { + putMany(b, p, blocks) + } +} // overlap1 gives the first 75 blocks to the first peer, and the last 75 blocks // to the second peer. This means both peers have the middle 50 blocks @@ -233,12 +324,8 @@ func overlap1(b *testing.B, provs []testinstance.Instance, blks []blocks.Block) bill := provs[0] jeff := provs[1] - if err := bill.Blockstore().PutMany(blks[:75]); err != nil { - b.Fatal(err) - } - if err := jeff.Blockstore().PutMany(blks[25:]); err != nil { - b.Fatal(err) - } + putMany(b, bill, blks[:75]) + putMany(b, jeff, blks[25:]) } // overlap2 gives every even numbered block to the first peer, odd numbered @@ -250,16 +337,16 @@ func overlap2(b *testing.B, provs []testinstance.Instance, blks []blocks.Block) bill := provs[0] jeff := provs[1] - bill.Blockstore().Put(blks[0]) - jeff.Blockstore().Put(blks[0]) + bill.Exchange.HasBlock(blks[0]) + jeff.Exchange.HasBlock(blks[0]) for i, blk := range blks { if i%3 == 0 { - bill.Blockstore().Put(blk) - jeff.Blockstore().Put(blk) + bill.Exchange.HasBlock(blk) + jeff.Exchange.HasBlock(blk) } else if i%2 == 1 { - bill.Blockstore().Put(blk) + bill.Exchange.HasBlock(blk) } else { - jeff.Blockstore().Put(blk) + jeff.Exchange.HasBlock(blk) } } } @@ -272,16 +359,16 @@ func overlap3(b *testing.B, provs []testinstance.Instance, blks []blocks.Block) bill := provs[0] jeff := provs[1] - bill.Blockstore().Put(blks[0]) - jeff.Blockstore().Put(blks[0]) + bill.Exchange.HasBlock(blks[0]) + jeff.Exchange.HasBlock(blks[0]) for i, blk := range blks { if i%3 == 0 { - bill.Blockstore().Put(blk) - jeff.Blockstore().Put(blk) + bill.Exchange.HasBlock(blk) + jeff.Exchange.HasBlock(blk) } else if i%2 == 1 { - bill.Blockstore().Put(blk) + bill.Exchange.HasBlock(blk) } else { - jeff.Blockstore().Put(blk) + jeff.Exchange.HasBlock(blk) } } } @@ -291,7 +378,18 @@ func overlap3(b *testing.B, provs []testinstance.Instance, blks []blocks.Block) // but we're mostly just testing performance of the sync algorithm func onePeerPerBlock(b *testing.B, provs []testinstance.Instance, blks []blocks.Block) { for _, blk := range blks { - provs[rand.Intn(len(provs))].Blockstore().Put(blk) + provs[rand.Intn(len(provs))].Exchange.HasBlock(blk) + } +} + +// randomRandom distributes each block randomly -- to random peers a random amount of times +func randomRandom(b *testing.B, provs []testinstance.Instance, blks []blocks.Block) { + for _, blk := range blks { + dups := rand.Intn(len(provs)) + 1 + perm := rand.Perm(len(provs)) + for i := 0; i < dups; i++ { + provs[perm[i]].Exchange.HasBlock(blk) + } } } diff --git a/go.mod b/go.mod index 1bafbdc6..b49fa738 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,10 @@ require ( github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-libp2p v0.1.1 github.com/libp2p/go-libp2p-core v0.0.3 + github.com/libp2p/go-libp2p-kad-dht v0.1.0 github.com/libp2p/go-libp2p-loggables v0.1.0 github.com/libp2p/go-libp2p-netutil v0.1.0 + github.com/libp2p/go-libp2p-record v0.1.0 github.com/libp2p/go-libp2p-testing v0.0.4 github.com/libp2p/go-msgio v0.0.4 github.com/mattn/go-colorable v0.1.2 // indirect diff --git a/go.sum b/go.sum index c80ee5bb..2b43f830 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= @@ -15,6 +17,7 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= @@ -33,12 +36,16 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= @@ -47,6 +54,7 @@ github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyF github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -94,6 +102,8 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.1 h1:+gPjbI+V3NktXZOqJA1kzbms2pYmhjgQQal0MzZrOAY= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= +github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= +github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -130,6 +140,7 @@ github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1 h1:52sB0TJuDk2nYMcMfHOKaPoaayDZjaYVCq6Vk1ejUTk= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= @@ -148,6 +159,10 @@ github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoA github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= +github.com/libp2p/go-libp2p-kad-dht v0.1.0 h1:cupGornmncdQCnjArAv9Txkqy43wgQCxoe/5pfIDF/4= +github.com/libp2p/go-libp2p-kad-dht v0.1.0/go.mod h1:EiH74cOsCxcu2Yh1XxQMpf1uHcLNm/hYEJBOIRlzHNE= +github.com/libp2p/go-libp2p-kbucket v0.2.0 h1:FB2a0VkOTNGTP5gu/I444u4WabNM9V1zCkQcWb7zajI= +github.com/libp2p/go-libp2p-kbucket v0.2.0/go.mod h1:JNymBToym3QXKBMKGy3m29+xprg0EVr/GJFHxFEdgh8= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -163,6 +178,8 @@ github.com/libp2p/go-libp2p-peerstore v0.1.0 h1:MKh7pRNPHSh1fLPj8u/M/s/napdmeNpo github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= +github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= +github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-secio v0.1.0 h1:NNP5KLxuP97sE5Bu3iuwOWyT/dKEGMN5zSLMWdB7GTQ= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-swarm v0.1.0 h1:HrFk2p0awrGEgch9JXK/qp/hfjqQfgNxpLWnCiWPg5s= @@ -276,6 +293,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -286,6 +306,8 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -296,15 +318,26 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -320,7 +353,16 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522 h1:bhOzK9QyoD0ogCnFro1m2mz41+Ib0oOhfJnBp5MR4K4= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -332,3 +374,4 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/network/ipfs_impl_test.go b/network/ipfs_impl_test.go index 2a8fab4c..774ee595 100644 --- a/network/ipfs_impl_test.go +++ b/network/ipfs_impl_test.go @@ -8,10 +8,9 @@ import ( bsmsg "github.com/ipfs/go-bitswap/message" tn "github.com/ipfs/go-bitswap/testnet" blocksutil "github.com/ipfs/go-ipfs-blocksutil" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-testing/net" + tnet "github.com/libp2p/go-libp2p-testing/net" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ) @@ -58,7 +57,7 @@ func TestMessageSendAndReceive(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() mn := mocknet.New(ctx) - mr := mockrouting.NewServer() + mr := tn.NewMockServer() streamNet, err := tn.StreamNet(ctx, mn, mr) if err != nil { t.Fatal("Unable to setup network") @@ -151,3 +150,104 @@ func TestMessageSendAndReceive(t *testing.T) { t.Fatal("Sent message blocks did not match received message blocks") } } + +func TestRealProviding(t *testing.T) { + // create network + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + mn := mocknet.New(ctx) + routing := tn.NewDHTServer() + streamNet, err := tn.StreamNet(ctx, mn, routing) + if err != nil { + t.Fatal("Unable to setup network") + } + p1 := tnet.RandIdentityOrFatal(t) + p2 := tnet.RandIdentityOrFatal(t) + p3 := tnet.RandIdentityOrFatal(t) + + bsnet1 := streamNet.Adapter(p1) + bsnet2 := streamNet.Adapter(p2) + bsnet3 := streamNet.Adapter(p3) + + r1 := &receiver{ + peers: make(map[peer.ID]struct{}), + messageReceived: make(chan struct{}), + connectionEvent: make(chan struct{}, 1), + } + r2 := &receiver{ + peers: make(map[peer.ID]struct{}), + messageReceived: make(chan struct{}), + connectionEvent: make(chan struct{}, 1), + } + r3 := &receiver{ + peers: make(map[peer.ID]struct{}), + messageReceived: make(chan struct{}), + connectionEvent: make(chan struct{}, 1), + } + bsnet1.SetDelegate(r1) + bsnet2.SetDelegate(r2) + bsnet3.SetDelegate(r3) + + mn.LinkAll() + bsnet1.ConnectTo(ctx, p2.ID()) + select { + case <-ctx.Done(): + t.Fatal("did not connect peer") + case <-r1.connectionEvent: + } + select { + case <-ctx.Done(): + t.Fatal("did not connect peer") + case <-r2.connectionEvent: + } + bsnet2.ConnectTo(ctx, p3.ID()) + select { + case <-ctx.Done(): + t.Fatal("did not connect peer") + case <-r2.connectionEvent: + } + select { + case <-ctx.Done(): + t.Fatal("did not connect peer") + case <-r3.connectionEvent: + } + if _, ok := r1.peers[p2.ID()]; !ok { + t.Fatal("did to connect to correct peer") + } + if _, ok := r2.peers[p1.ID()]; !ok { + t.Fatal("did to connect to correct peer") + } + if _, ok := r2.peers[p3.ID()]; !ok { + t.Fatal("did to connect to correct peer") + } + if _, ok := r3.peers[p2.ID()]; !ok { + t.Fatal("did to connect to correct peer") + } + blockGenerator := blocksutil.NewBlockGenerator() + block1 := blockGenerator.Next() + // provide on node 3 + // DHT can setup routing table in seperate thread, + // meaning it's not finished even bitswap peers are connected + // so attempt a few times with delay +provideLoop: + for i := 0; i < 5; i++ { + err = bsnet3.Provide(ctx, block1.Cid()) + if err == nil { + break provideLoop + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatalf("Unable to provide node: %s", err) + } + incomingProvders := bsnet1.FindProvidersAsync(ctx, block1.Cid(), 1) + select { + case <-ctx.Done(): + t.Fatal("Unable to find provided block") + case prov := <-incomingProvders: + if prov != p3.ID() { + t.Fatal("Found incorrect provider") + } + } +} diff --git a/testnet/interface.go b/testnet/interface.go index b6616256..faa2fe7d 100644 --- a/testnet/interface.go +++ b/testnet/interface.go @@ -4,7 +4,7 @@ import ( bsnet "github.com/ipfs/go-bitswap/network" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-testing/net" + tnet "github.com/libp2p/go-libp2p-testing/net" ) // Network is an interface for generating bitswap network interfaces diff --git a/testnet/peernet.go b/testnet/peernet.go index ffbe1026..5b18bad5 100644 --- a/testnet/peernet.go +++ b/testnet/peernet.go @@ -4,22 +4,69 @@ import ( "context" bsnet "github.com/ipfs/go-bitswap/network" + mockrouting "github.com/ipfs/go-ipfs-routing/mock" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" + dht "github.com/libp2p/go-libp2p-kad-dht" + opts "github.com/libp2p/go-libp2p-kad-dht/opts" + record "github.com/libp2p/go-libp2p-record" + tnet "github.com/libp2p/go-libp2p-testing/net" ds "github.com/ipfs/go-datastore" - mockrouting "github.com/ipfs/go-ipfs-routing/mock" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-testing/net" mockpeernet "github.com/libp2p/go-libp2p/p2p/net/mock" ) +// RoutingServer can generate routing clients from a host and an indentity +type RoutingServer interface { + Client(ctx context.Context, p2pHost host.Host, p tnet.Identity) routing.Routing +} + +// MockServer returns clients that use mock routing. +type MockServer struct { + rs mockrouting.Server +} + +// NewMockServer returns a MockServer. +func NewMockServer() RoutingServer { + return &MockServer{rs: mockrouting.NewServer()} +} + +// Client returns a new mock routing instance for the given host and identity. +func (ms *MockServer) Client(ctx context.Context, p2pHost host.Host, p tnet.Identity) routing.Routing { + return ms.rs.ClientWithDatastore(ctx, p, ds.NewMapDatastore()) +} + +// DHTServer generates real Kademlia DHT routing instances. +type DHTServer struct { +} + +// NewDHTServer generates a new DHT based routing server +func NewDHTServer() RoutingServer { + return &DHTServer{} +} + +// Client returns a new Kademlia DHT Routing interface +func (dhtServer *DHTServer) Client(ctx context.Context, p2pHost host.Host, p tnet.Identity) routing.Routing { + routing, err := dht.New( + ctx, p2pHost, + opts.Datastore(ds.NewMapDatastore()), + opts.Validator(record.PublicKeyValidator{}), + ) + if err != nil { + panic(err.Error()) + } + return routing +} + type peernet struct { mockpeernet.Mocknet - routingserver mockrouting.Server + routingserver RoutingServer } // StreamNet is a testnet that uses libp2p's MockNet -func StreamNet(ctx context.Context, net mockpeernet.Mocknet, rs mockrouting.Server) (Network, error) { +func StreamNet(ctx context.Context, net mockpeernet.Mocknet, rs RoutingServer) (Network, error) { return &peernet{net, rs}, nil } @@ -28,7 +75,7 @@ func (pn *peernet) Adapter(p tnet.Identity) bsnet.BitSwapNetwork { if err != nil { panic(err.Error()) } - routing := pn.routingserver.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore()) + routing := pn.routingserver.Client(context.TODO(), client, p) return bsnet.NewFromIpfsHost(client, routing) }