From a8ce97e64b7c4e5cf4d13c4bacdf58fffbfcc119 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 8 Jan 2019 02:58:48 +0100 Subject: [PATCH 1/8] swarm/network: fix skipped tests related to suggestPeer --- swarm/network/kademlia.go | 35 ++++++++++++------- swarm/network/kademlia_test.go | 21 +++-------- .../simulations/discovery/discovery_test.go | 12 +++---- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index da99287f1cc3..e46d36c4b072 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -175,7 +175,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { k.lock.Lock() defer k.lock.Unlock() minsize := k.MinBinSize - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) + depth := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected var ppo int @@ -404,23 +404,16 @@ func (k *Kademlia) NeighbourhoodDepth() (depth int) { return depthForPot(k.conns, k.NeighbourhoodSize, k.base) } -// depthForPot returns the proximity order that defines the distance of -// the nearest neighbour set with cardinality >= NeighbourhoodSize -// if there is altogether less than NeighbourhoodSize peers it returns 0 +// neighbourhoodRadiusForPot returns the proximity order that defines the distance of +// the nearest neighbour set with cardinality >= MinProxBinSize +// if there is altogether less than MinProxBinSize peers it returns 0 // caller must hold the lock -func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { +func neighbourhoodRadiusForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { if p.Size() <= neighbourhoodSize { return 0 } - // total number of peers in iteration var size int - - // determining the depth is a two-step process - // first we find the proximity bin of the shallowest of the NeighbourhoodSize peers - // the numeric value of depth cannot be higher than this - var maxDepth int - f := func(v pot.Val, i int) bool { // po == 256 means that addr is the pivot address(self) if i == 256 { @@ -431,13 +424,26 @@ func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int // this means we have all nn-peers. // depth is by default set to the bin of the farthest nn-peer if size == neighbourhoodSize { - maxDepth = i + depth = i return false } return true } p.EachNeighbour(pivotAddr, Pof, f) + return depth +} + +// depthForPot returns the depth for the pot +// caller must hold the lock +func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) { + if p.Size() <= minProxBinSize { + return 0 + } + // determining the depth is a two-step process + // first we find the proximity bin of the shallowest of the MinProxBinSize peers + // the numeric value of depth cannot be higher than this + maxDepth := neighbourhoodRadiusForPot(p, minProxBinSize, pivotAddr) // the second step is to test for empty bins in order from shallowest to deepest // if an empty bin is found, this will be the actual depth @@ -745,6 +751,9 @@ type Health struct { func (k *Kademlia) Healthy(pp *PeerPot) *Health { k.lock.RLock() defer k.lock.RUnlock() + if len(pp.NNSet) < k.MinProxBinSize { + panic("wrong peerpot") + } gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet) depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index fcb277fde788..32f2e914f056 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -170,18 +170,18 @@ func TestHealthStrict(t *testing.T) { // no peers // unhealthy (and lonely) k := newTestKademlia("11111111") - assertHealth(t, k, false, false) + // assertHealth(t, k, false, false) // know one peer but not connected // unhealthy Register(k, "11100000") log.Trace(k.String()) - assertHealth(t, k, false, false) + // assertHealth(t, k, false, false) // know one peer and connected // healthy On(k, "11100000") - assertHealth(t, k, true, false) + // assertHealth(t, k, true, false) // know two peers, only one connected // unhealthy @@ -292,8 +292,6 @@ func TestSuggestPeerBug(t *testing.T) { } func TestSuggestPeerFindPeers(t *testing.T) { - t.Skip("The SuggestPeers implementation seems to have weaknesses exposed by the change in the new depth calculation. The results are no longer predictable") - testnum := 0 // test 0 // 2 row gap, unsaturated proxbin, no callables -> want PO 0 @@ -356,7 +354,7 @@ func TestSuggestPeerFindPeers(t *testing.T) { // with reasonably set Interval log.Trace("foo") log.Trace(k.String()) - err = testSuggestPeer(k, "", 1, false) + err = testSuggestPeer(k, "", 1, true) if err != nil { t.Fatalf("%d %v", testnum, err.Error()) } @@ -615,8 +613,6 @@ func TestKademliaHiveString(t *testing.T) { // the SuggestPeer and Healthy methods for provided hex-encoded addresses. // Argument pivotAddr is the address of the kademlia. func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { - - t.Skip("this test relies on SuggestPeer which is now not reliable. See description in TestSuggestPeerFindPeers") addr := common.Hex2Bytes(pivotAddr) var byteAddrs [][]byte for _, ahex := range addrs { @@ -636,10 +632,6 @@ func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { } } - ppmap := NewPeerPotMap(k.NeighbourhoodSize, byteAddrs) - - pp := ppmap[pivotAddr] - for { a, _, _ := k.SuggestPeer() if a == nil { @@ -648,10 +640,7 @@ func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { k.On(NewPeer(&BzzPeer{BzzAddr: a}, k)) } - h := k.Healthy(pp) - if !(h.ConnectNN && h.KnowNN && h.CountKnowNN > 0) { - t.Fatalf("not healthy: %#v\n%v", h, k.String()) - } + assertHealth(t, k, true, true) } /* diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 7d03789870c5..d5435768ac1f 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -151,7 +151,6 @@ func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { } func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { - t.Skip("discovery tests depend on suggestpeer, which is unreliable after kademlia depth change.") startedAt := time.Now() result, err := discoverySimulation(nodes, conns, adapter) if err != nil { @@ -179,7 +178,6 @@ func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.No } func testDiscoveryPersistenceSimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) map[int][]byte { - t.Skip("discovery tests depend on suggestpeer, which is unreliable after kademlia depth change.") persistenceEnabled = true discoveryEnabled = true @@ -269,7 +267,8 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul } healthy := &network.Health{} - if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { + if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { + // if err := client.Call(&healthy, "hive_healthy", ppmap[id2addr[id]); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } log.Info(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive)) @@ -353,8 +352,8 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt } healthy := &network.Health{} addr := id.String() - ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) - if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { + ppmap := network.NewPeerPotMap(network.NewKadParams().MinProxBinSize, addrs) + if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { return fmt.Errorf("error getting node health: %s", err) } @@ -427,7 +426,8 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt healthy := &network.Health{} ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) - if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { + if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { + // if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v", id, healthy.ConnectNN, healthy.KnowNN)) From 408fe368325ee3b19322766377324003315c6733 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 12 Jan 2019 05:07:19 +0100 Subject: [PATCH 2/8] swarm/network: rename depth to radius --- swarm/network/kademlia.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index e46d36c4b072..5422afc098b3 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -175,12 +175,12 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { k.lock.Lock() defer k.lock.Unlock() minsize := k.MinBinSize - depth := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) + radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected var ppo int k.addrs.EachNeighbour(k.base, Pof, func(val pot.Val, po int) bool { - if po < depth { + if po < radius { return false } e := val.(*entry) @@ -208,7 +208,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { bpo = append(bpo, po) minsize = size } - return size > 0 && po < depth + return size > 0 && po < radius }) // all buckets are full, ie., minsize == k.MinBinSize if len(bpo) == 0 { @@ -221,7 +221,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { nxt := bpo[0] k.addrs.EachBin(k.base, Pof, nxt, func(po, _ int, f func(func(pot.Val) bool) bool) bool { // for each bin (up until depth) we find callable candidate peers - if po >= depth { + if po >= radius { return false } return f(func(val pot.Val) bool { @@ -751,7 +751,7 @@ type Health struct { func (k *Kademlia) Healthy(pp *PeerPot) *Health { k.lock.RLock() defer k.lock.RUnlock() - if len(pp.NNSet) < k.MinProxBinSize { + if len(pp.NNSet) < k.NeighbourhoodSize { panic("wrong peerpot") } gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) From 28347e078086e7ba3ee46fb92d25d9e2fc5d2131 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 12 Jan 2019 05:59:08 +0100 Subject: [PATCH 3/8] swarm/network: uncomment assertHealth and improve comments --- swarm/network/kademlia.go | 35 ++++++++++--------- swarm/network/kademlia_test.go | 6 ++-- .../simulations/discovery/discovery_test.go | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 5422afc098b3..4d9dcd3624e2 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -168,15 +168,13 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { return nil } -// SuggestPeer returns a known peer for the lowest proximity bin for the -// lowest bincount below depth -// naturally if there is an empty row it returns a peer for that +// SuggestPeer returns an unconnected peer address as a peer suggestion for connection func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { k.lock.Lock() defer k.lock.Unlock() minsize := k.MinBinSize radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) - // if there is a callable neighbour within the current proxBin, connect + // finds a callable neighbour within the current neighbourhood radius // this makes sure nearest neighbour set is fully connected var ppo int k.addrs.EachNeighbour(k.base, Pof, func(val pot.Val, po int) bool { @@ -196,6 +194,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { return a, 0, false } + // if there are no callable neighbours, find the undersaturated bins from shallow to deep var bpo []int prev := -1 k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { @@ -210,13 +209,10 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { } return size > 0 && po < radius }) - // all buckets are full, ie., minsize == k.MinBinSize + // all buckets are saturated, ie., minsize >= k.MinBinSize, no peer suggested if len(bpo) == 0 { return nil, 0, false } - // as long as we got candidate peers to connect to - // dont ask for new peers (want = false) - // try to select a candidate peer // find the first callable peer nxt := bpo[0] k.addrs.EachBin(k.base, Pof, nxt, func(po, _ int, f func(func(pot.Val) bool) bool) bool { @@ -398,15 +394,18 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { }) } +// NeighbourhoodDepth returns the depth for the pot, see depthForPot func (k *Kademlia) NeighbourhoodDepth() (depth int) { k.lock.RLock() defer k.lock.RUnlock() return depthForPot(k.conns, k.NeighbourhoodSize, k.base) } -// neighbourhoodRadiusForPot returns the proximity order that defines the distance of -// the nearest neighbour set with cardinality >= MinProxBinSize -// if there is altogether less than MinProxBinSize peers it returns 0 +// neighbourhoodRadiusForPot returns the neighbourhood radius of the kademlia +// neighbourhood radius encloses the nearest neighbour set with cardinality >= neighbourhoodSize +// i.e., neighbourhood radius is the deepest PO such that all bins not shallower altogether +// contain at least neighbourhoodSize connected peers +// if there is altogether less than neighbourhoodSize peers connected, it returns 0 // caller must hold the lock func neighbourhoodRadiusForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { if p.Size() <= neighbourhoodSize { @@ -435,15 +434,19 @@ func neighbourhoodRadiusForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []by } // depthForPot returns the depth for the pot +// depth is the radius of the minimal extension of nearest neighbourhood that +// includes all empty PO bins. I.e., depth is the deepest PO such that +// - it is not deeper than neighbourhood radius +// - all bins shallower than depth are not empty // caller must hold the lock -func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) { - if p.Size() <= minProxBinSize { +func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { + if p.Size() <= neighbourhoodSize { return 0 } // determining the depth is a two-step process - // first we find the proximity bin of the shallowest of the MinProxBinSize peers + // first we find the proximity bin of the shallowest of the neighbourhoodSize peers // the numeric value of depth cannot be higher than this - maxDepth := neighbourhoodRadiusForPot(p, minProxBinSize, pivotAddr) + maxDepth := neighbourhoodRadiusForPot(p, neighbourhoodSize, pivotAddr) // the second step is to test for empty bins in order from shallowest to deepest // if an empty bin is found, this will be the actual depth @@ -752,7 +755,7 @@ func (k *Kademlia) Healthy(pp *PeerPot) *Health { k.lock.RLock() defer k.lock.RUnlock() if len(pp.NNSet) < k.NeighbourhoodSize { - panic("wrong peerpot") + log.Warn("peerpot NNSet < NeighbourhoodSize") } gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 32f2e914f056..a096e7b44d91 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -170,18 +170,18 @@ func TestHealthStrict(t *testing.T) { // no peers // unhealthy (and lonely) k := newTestKademlia("11111111") - // assertHealth(t, k, false, false) + assertHealth(t, k, false, false) // know one peer but not connected // unhealthy Register(k, "11100000") log.Trace(k.String()) - // assertHealth(t, k, false, false) + assertHealth(t, k, false, false) // know one peer and connected // healthy On(k, "11100000") - // assertHealth(t, k, true, false) + assertHealth(t, k, true, false) // know two peers, only one connected // unhealthy diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index d5435768ac1f..3daf9650d82c 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -352,7 +352,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt } healthy := &network.Health{} addr := id.String() - ppmap := network.NewPeerPotMap(network.NewKadParams().MinProxBinSize, addrs) + ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { return fmt.Errorf("error getting node health: %s", err) } From bd9014da6979111a3b4514eb2fa6b27db08373fd Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 12 Jan 2019 06:04:29 +0100 Subject: [PATCH 4/8] swarm/network: remove commented code --- swarm/network/simulations/discovery/discovery_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 3daf9650d82c..e13517a63eaf 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -268,7 +268,6 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul healthy := &network.Health{} if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { - // if err := client.Call(&healthy, "hive_healthy", ppmap[id2addr[id]); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } log.Info(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive)) @@ -427,7 +426,6 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { - // if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v", id, healthy.ConnectNN, healthy.KnowNN)) From df0928bfafbcaf71f62fea5d6bae5bf2cbf5a58e Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 12 Jan 2019 14:49:02 +0100 Subject: [PATCH 5/8] swarm/network: kademlia suggestPeer algo correction --- swarm/network/kademlia.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 4d9dcd3624e2..997db08a6732 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -214,12 +214,24 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { return nil, 0, false } // find the first callable peer - nxt := bpo[0] - k.addrs.EachBin(k.base, Pof, nxt, func(po, _ int, f func(func(pot.Val) bool) bool) bool { - // for each bin (up until depth) we find callable candidate peers + // for each bin (up until neighbourhood radius) we find callable candidate peers + // nxt := bpo[0] + ci := 0 + saturationDepth := bpo[0] + cpo := saturationDepth + k.addrs.EachBin(k.base, Pof, cpo, func(po, _ int, f func(func(pot.Val) bool) bool) bool { if po >= radius { return false } + if po < cpo { + return true + } + for ; ci < len(bpo) && cpo < po; ci++ { + cpo = bpo[ci] + } + if ci == len(bpo) { + return false + } return f(func(val pot.Val) bool { e := val.(*entry) c := k.callable(e) @@ -235,11 +247,11 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { } // no candidate peer found, request for the short bin var changed bool - if uint8(nxt) < k.depth { - k.depth = uint8(nxt) + if uint8(saturationDepth) < k.depth { + k.depth = uint8(saturationDepth) changed = true } - return a, nxt, changed + return a, saturationDepth, changed } // On inserts the peer as a kademlia peer into the live peers From 4ceac790698c6baa77bf50fb261326c554b25cf7 Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 14 Jan 2019 08:50:51 +0100 Subject: [PATCH 6/8] swarm/network: kademlia suggest peer * simplify suggest Peer code * improve peer suggestion algo * add comments * kademlia testing improvements * assertHealth -> checkHealth (test helper) * testSuggestPeer -> checkSuggestPeer (test helper) * remove testSuggestPeerBug and TestKademliaCase --- swarm/network/kademlia.go | 169 +++-- swarm/network/kademlia_test.go | 711 ++++-------------- .../simulations/discovery/discovery_test.go | 2 +- 3 files changed, 243 insertions(+), 639 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 997db08a6732..a3c5482ea397 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -169,89 +169,109 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { } // SuggestPeer returns an unconnected peer address as a peer suggestion for connection -func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { +func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, changed bool) { k.lock.Lock() defer k.lock.Unlock() - minsize := k.MinBinSize radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) - // finds a callable neighbour within the current neighbourhood radius - // this makes sure nearest neighbour set is fully connected - var ppo int - k.addrs.EachNeighbour(k.base, Pof, func(val pot.Val, po int) bool { - if po < radius { - return false - } - e := val.(*entry) - c := k.callable(e) - if c { - a = e.BzzAddr - } - ppo = po - return !c - }) - if a != nil { - log.Trace(fmt.Sprintf("%08x candidate nearest neighbour found: %v (%v)", k.BaseAddr()[:4], a, ppo)) - return a, 0, false - } - - // if there are no callable neighbours, find the undersaturated bins from shallow to deep - var bpo []int - prev := -1 + // find the undersaturated bins from lowest cardinality from shallow to deep + // first map unsaturated bins to cardinalities + spo := make(map[int][]int) + var prev int + saturationDepth = -1 + var pastDepth bool k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { - prev++ + // process skipped empty bins for ; prev < po; prev++ { - bpo = append(bpo, prev) - minsize = 0 + // find the lowest unsaturated bin + if saturationDepth == -1 { + saturationDepth = prev + } + // if there is an empty bin, depth is surely passed + pastDepth = true + spo[0] = append(spo[0], prev) + } + prev = po + 1 + // past radius, depth is surely passed + if po >= radius { + pastDepth = true + } + // beyond depth the bin is treated as unsaturated to achieve full connectivity + // to all neighbours + if pastDepth && size >= k.MinBinSize { + size = k.MinBinSize - 1 } - if size < minsize { - bpo = append(bpo, po) - minsize = size + // process non-empty unsaturated bins + if size < k.MinBinSize { + // find the lowest unsaturated bin + if saturationDepth == -1 { + saturationDepth = po + } + spo[size] = append(spo[size], po) } - return size > 0 && po < radius + return true + }) + // to trigger peer requests for peers closer than closest connection, include + // all bins from nearest connection upto nearest address as unsaturated + var nearestAddrAt int + k.addrs.EachNeighbour(k.base, Pof, func(_ pot.Val, po int) bool { + nearestAddrAt = po + return false }) - // all buckets are saturated, ie., minsize >= k.MinBinSize, no peer suggested - if len(bpo) == 0 { + for ; prev <= nearestAddrAt; prev++ { + spo[0] = append(spo[0], prev) + } + // all PO bins are saturated, ie., minsize >= k.MinBinSize, no peer suggested + if len(spo) == 0 { return nil, 0, false } // find the first callable peer + // starting from the lowest cardinality // for each bin (up until neighbourhood radius) we find callable candidate peers - // nxt := bpo[0] - ci := 0 - saturationDepth := bpo[0] - cpo := saturationDepth - k.addrs.EachBin(k.base, Pof, cpo, func(po, _ int, f func(func(pot.Val) bool) bool) bool { - if po >= radius { - return false + for i := 0; i < k.MinBinSize && suggestedPeer == nil; i++ { + bins, ok := spo[i] + if !ok { + continue } - if po < cpo { - return true - } - for ; ci < len(bpo) && cpo < po; ci++ { - cpo = bpo[ci] - } - if ci == len(bpo) { - return false - } - return f(func(val pot.Val) bool { - e := val.(*entry) - c := k.callable(e) - if c { - a = e.BzzAddr + ci := 0 + cpo := bins[0] + k.addrs.EachBin(k.base, Pof, cpo, func(po, _ int, f func(func(pot.Val) bool) bool) bool { + cpo = bins[ci] + // find the next bin with cardinality i + if cpo == po { + ci++ + } else { + // skip bins that have no addresses + for ; ci < len(bins) && cpo < po; ci++ { + cpo = bins[ci] + } + if po < cpo { + ci-- + return true + } + // stop if there are no addresses + if cpo < po { + return false + } } - return !c + // find a callable peer out of the addresses in the unsaturated bin + // stop if found + f(func(val pot.Val) bool { + e := val.(*entry) + c := k.callable(e) + if c { + suggestedPeer = e.BzzAddr + } + return !c + }) + return ci < len(bins) && suggestedPeer == nil }) - }) - // found a candidate - if a != nil { - return a, 0, false } - // no candidate peer found, request for the short bin - var changed bool + if uint8(saturationDepth) < k.depth { k.depth = uint8(saturationDepth) - changed = true + return suggestedPeer, saturationDepth, true } - return a, saturationDepth, changed + return suggestedPeer, 0, false } // On inserts the peer as a kademlia peer into the live peers @@ -648,23 +668,20 @@ func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot { return ppmap } -// saturation iterates through all peers and -// returns the smallest po value in which the node has less than n peers -// if the iterator reaches depth, then value for depth is returned -// TODO move to separate testing tools file -// TODO this function will stop at the first bin with less than MinBinSize peers, even if there are empty bins between that bin and the depth. This may not be correct behavior +// saturation returns the smallest po value in which the node has less than MinBinSize peers +// if the iterator reaches neighbourhood radius, then the last bin + 1 is returned func (k *Kademlia) saturation() int { prev := -1 - k.addrs.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { + radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) + k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { prev++ + if po >= radius { + return false + } return prev == po && size >= k.MinBinSize }) - // TODO evaluate whether this check cannot just as well be done within the eachbin - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) - - // if in the iterator above we iterated deeper than the neighbourhood depth - return depth - if depth < prev { - return depth + if prev < 0 { + return 0 } return prev } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index a096e7b44d91..8a724756b646 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -17,7 +17,6 @@ package network import ( - "bytes" "fmt" "os" "testing" @@ -43,39 +42,46 @@ func testKadPeerAddr(s string) *BzzAddr { func newTestKademliaParams() *KadParams { params := NewKadParams() - // TODO why is this 1? - params.MinBinSize = 1 + params.MinBinSize = 2 params.NeighbourhoodSize = 2 return params } -func newTestKademlia(b string) *Kademlia { +type testKademlia struct { + *Kademlia + t *testing.T +} + +func newTestKademlia(t *testing.T, b string) *testKademlia { base := pot.NewAddressFromString(b) - return NewKademlia(base, newTestKademliaParams()) + return &testKademlia{ + Kademlia: NewKademlia(base, newTestKademliaParams()), + t: t, + } } -func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k) +func (tk *testKademlia) newTestKadPeer(s string, lightNode bool) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, tk.Kademlia) } -func On(k *Kademlia, ons ...string) { +func (tk *testKademlia) On(ons ...string) { for _, s := range ons { - k.On(newTestKadPeer(k, s, false)) + tk.Kademlia.On(tk.newTestKadPeer(s, false)) } } -func Off(k *Kademlia, offs ...string) { +func (tk *testKademlia) Off(offs ...string) { for _, s := range offs { - k.Off(newTestKadPeer(k, s, false)) + tk.Kademlia.Off(tk.newTestKadPeer(s, false)) } } -func Register(k *Kademlia, regs ...string) { +func (tk *testKademlia) Register(regs ...string) { var as []*BzzAddr for _, s := range regs { as = append(as, testKadPeerAddr(s)) } - err := k.Register(as...) + err := tk.Kademlia.Register(as...) if err != nil { panic(err.Error()) } @@ -169,76 +175,71 @@ func TestHealthStrict(t *testing.T) { // base address is all zeros // no peers // unhealthy (and lonely) - k := newTestKademlia("11111111") - assertHealth(t, k, false, false) + tk := newTestKademlia(t, "11111111") + tk.checkHealth(false, false) // know one peer but not connected // unhealthy - Register(k, "11100000") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11100000") + tk.checkHealth(false, false) // know one peer and connected // healthy - On(k, "11100000") - assertHealth(t, k, true, false) + tk.On("11100000") + tk.checkHealth(true, false) // know two peers, only one connected // unhealthy - Register(k, "11111100") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11111100") + tk.checkHealth(false, false) // know two peers and connected to both // healthy - On(k, "11111100") - assertHealth(t, k, true, false) + tk.On("11111100") + tk.checkHealth(true, false) // know three peers, connected to the two deepest // healthy - Register(k, "00000000") - log.Trace(k.String()) - assertHealth(t, k, true, false) + tk.Register("00000000") + tk.checkHealth(true, false) // know three peers, connected to all three // healthy - On(k, "00000000") - assertHealth(t, k, true, false) + tk.On("00000000") + tk.checkHealth(true, false) // add fourth peer deeper than current depth // unhealthy - Register(k, "11110000") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11110000") + tk.checkHealth(false, false) // connected to three deepest peers // healthy - On(k, "11110000") - assertHealth(t, k, true, false) + tk.On("11110000") + tk.checkHealth(true, false) // add additional peer in same bin as deepest peer // unhealthy - Register(k, "11111101") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11111101") + tk.checkHealth(false, false) // four deepest of five peers connected // healthy - On(k, "11111101") - assertHealth(t, k, true, false) + tk.On("11111101") + tk.checkHealth(true, false) } -func assertHealth(t *testing.T, k *Kademlia, expectHealthy bool, expectSaturation bool) { - t.Helper() - kid := common.Bytes2Hex(k.BaseAddr()) - addrs := [][]byte{k.BaseAddr()} - k.EachAddr(nil, 255, func(addr *BzzAddr, po int) bool { +func (tk *testKademlia) checkHealth(expectHealthy bool, expectSaturation bool) { + tk.t.Helper() + kid := common.Bytes2Hex(tk.BaseAddr()) + addrs := [][]byte{tk.BaseAddr()} + tk.EachAddr(nil, 255, func(addr *BzzAddr, po int) bool { addrs = append(addrs, addr.Address()) return true }) - pp := NewPeerPotMap(k.NeighbourhoodSize, addrs) - healthParams := k.Healthy(pp[kid]) + pp := NewPeerPotMap(tk.NeighbourhoodSize, addrs) + healthParams := tk.Healthy(pp[kid]) // definition of health, all conditions but be true: // - we at least know one peer @@ -246,23 +247,23 @@ func assertHealth(t *testing.T, k *Kademlia, expectHealthy bool, expectSaturatio // - we are connected to all known neighbors health := healthParams.KnowNN && healthParams.ConnectNN && healthParams.CountKnowNN > 0 if expectHealthy != health { - t.Fatalf("expected kademlia health %v, is %v\n%v", expectHealthy, health, k.String()) + tk.t.Fatalf("expected kademlia health %v, is %v\n%v", expectHealthy, health, tk.String()) } } -func testSuggestPeer(k *Kademlia, expAddr string, expPo int, expWant bool) error { - addr, o, want := k.SuggestPeer() - log.Trace("suggestpeer return", "a", addr, "o", o, "want", want) +func (tk *testKademlia) checkSuggestPeer(expAddr string, expDepth int, expChanged bool) { + tk.t.Helper() + addr, depth, changed := tk.SuggestPeer() + log.Trace("suggestPeer return", "addr", addr, "depth", depth, "changed", changed) if binStr(addr) != expAddr { - return fmt.Errorf("incorrect peer address suggested. expected %v, got %v", expAddr, binStr(addr)) + tk.t.Fatalf("incorrect peer address suggested. expected %v, got %v", expAddr, binStr(addr)) } - if o != expPo { - return fmt.Errorf("incorrect prox order suggested. expected %v, got %v", expPo, o) + if depth != expDepth { + tk.t.Fatalf("incorrect saturation depth suggested. expected %v, got %v", expDepth, depth) } - if want != expWant { - return fmt.Errorf("expected SuggestPeer to want peers: %v", expWant) + if changed != expChanged { + tk.t.Fatalf("expected depth change = %v, got %v", expChanged, changed) } - return nil } func binStr(a *BzzAddr) string { @@ -272,598 +273,184 @@ func binStr(a *BzzAddr) string { return pot.ToBin(a.Address())[:8] } -// TODO explain why this bug occurred and how it should have been mitigated -func TestSuggestPeerBug(t *testing.T) { - // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("00000000") - On(k, - "10000000", "11000000", - "01000000", - - "00010000", "00011000", - ) - Off(k, - "01000000", - ) - err := testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } -} - func TestSuggestPeerFindPeers(t *testing.T) { - testnum := 0 - // test 0 - // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("00000000") - On(k, "00100000") - err := testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk := newTestKademlia(t, "00000000") + tk.On("00100000") + tk.checkSuggestPeer("", 0, false) - // test 1 - // 2 row gap, saturated proxbin, no callables -> want PO 0 - On(k, "00010000") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("00010000") + tk.checkSuggestPeer("", 0, false) - // test 2 - // 1 row gap (1 less), saturated proxbin, no callables -> want PO 1 - On(k, "10000000") - err = testSuggestPeer(k, "", 1, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("10000000", "10000001") + tk.checkSuggestPeer("", 0, false) - // test 3 - // no gap (1 less), saturated proxbin, no callables -> do not want more - On(k, "01000000", "00100001") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000") + tk.Off("10000001") + tk.checkSuggestPeer("10000001", 0, true) - // test 4 - // oversaturated proxbin, > do not want more - On(k, "00100001") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("00100001") + tk.Off("01000000") + tk.checkSuggestPeer("01000000", 0, false) - // test 5 - // reintroduce gap, disconnected peer callable - Off(k, "01000000") - log.Trace(k.String()) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ - - // test 6 // second time disconnected peer not callable // with reasonably set Interval - log.Trace("foo") - log.Trace(k.String()) - err = testSuggestPeer(k, "", 1, true) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.checkSuggestPeer("", 0, false) - // test 6 // on and off again, peer callable again - On(k, "01000000") - Off(k, "01000000") - log.Trace(k.String()) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000") + tk.Off("01000000") + tk.checkSuggestPeer("01000000", 0, false) - // test 7 - // new closer peer appears, it is immediately wanted - On(k, "01000000") - Register(k, "00010001") - err = testSuggestPeer(k, "00010001", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ - - // test 8 - // PO1 disconnects - On(k, "00010001") - log.Info(k.String()) - Off(k, "01000000") - log.Info(k.String()) - // second time, gap filling - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000", "10000001") + tk.checkSuggestPeer("", 0, false) - // test 9 - On(k, "01000000") - log.Info(k.String()) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("00010001") + tk.checkSuggestPeer("00010001", 0, false) - // test 10 - k.MinBinSize = 2 - log.Info(k.String()) - err = testSuggestPeer(k, "", 0, true) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("00010001") + tk.Off("01000000") + tk.checkSuggestPeer("01000000", 0, false) - // test 11 - Register(k, "01000001") - log.Info(k.String()) - err = testSuggestPeer(k, "01000001", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000") + tk.checkSuggestPeer("", 0, false) - // test 12 - On(k, "10000001") - log.Trace(fmt.Sprintf("Kad:\n%v", k.String())) - err = testSuggestPeer(k, "", 1, true) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("01000001") + tk.checkSuggestPeer("01000001", 0, false) - // test 13 - On(k, "01000001") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000001") + tk.checkSuggestPeer("", 0, false) - // test 14 - k.MinBinSize = 3 - Register(k, "10000010") - err = testSuggestPeer(k, "10000010", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("10000010", "01000010", "00100010") + tk.checkSuggestPeer("", 0, false) - // test 15 - On(k, "10000010") - err = testSuggestPeer(k, "", 1, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("00010010") + tk.checkSuggestPeer("00010010", 0, false) - // test 16 - On(k, "01000010") - err = testSuggestPeer(k, "", 2, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Off("00100001") + tk.checkSuggestPeer("00100010", 2, true) - // test 17 - On(k, "00100010") - err = testSuggestPeer(k, "", 3, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Off("01000001") + tk.checkSuggestPeer("01000010", 1, true) - // test 18 - On(k, "00010010") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.checkSuggestPeer("01000001", 0, false) + tk.checkSuggestPeer("00100001", 0, false) + tk.checkSuggestPeer("", 0, false) + + tk.On("01000001", "00100001") + tk.Register("10000100", "01000100", "00100100") + tk.Register("00000100", "00000101", "00000110") + tk.Register("00000010", "00000011", "00000001") + + tk.checkSuggestPeer("00000110", 0, false) + tk.checkSuggestPeer("00000101", 0, false) + tk.checkSuggestPeer("00000100", 0, false) + tk.checkSuggestPeer("00000011", 0, false) + tk.checkSuggestPeer("00000010", 0, false) + tk.checkSuggestPeer("00000001", 0, false) + tk.checkSuggestPeer("", 0, false) } // a node should stay in the address book if it's removed from the kademlia func TestOffEffectingAddressBookNormalNode(t *testing.T) { - k := newTestKademlia("00000000") + tk := newTestKademlia(t, "00000000") // peer added to kademlia - k.On(newTestKadPeer(k, "01000000", false)) + tk.On("01000000") // peer should be in the address book - if k.addrs.Size() != 1 { + if tk.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should be among live connections - if k.conns.Size() != 1 { + if tk.conns.Size() != 1 { t.Fatal("live peers should contain 1 entry") } // remove peer from kademlia - k.Off(newTestKadPeer(k, "01000000", false)) + tk.Off("01000000") // peer should be in the address book - if k.addrs.Size() != 1 { + if tk.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should not be among live connections - if k.conns.Size() != 0 { + if tk.conns.Size() != 0 { t.Fatal("live peers should contain 0 entry") } } // a light node should not be in the address book func TestOffEffectingAddressBookLightNode(t *testing.T) { - k := newTestKademlia("00000000") + tk := newTestKademlia(t, "00000000") // light node peer added to kademlia - k.On(newTestKadPeer(k, "01000000", true)) + tk.Kademlia.On(tk.newTestKadPeer("01000000", true)) // peer should not be in the address book - if k.addrs.Size() != 0 { + if tk.addrs.Size() != 0 { t.Fatal("known peer addresses should contain 0 entry") } // peer should be among live connections - if k.conns.Size() != 1 { + if tk.conns.Size() != 1 { t.Fatal("live peers should contain 1 entry") } // remove peer from kademlia - k.Off(newTestKadPeer(k, "01000000", true)) + tk.Kademlia.Off(tk.newTestKadPeer("01000000", true)) // peer should not be in the address book - if k.addrs.Size() != 0 { + if tk.addrs.Size() != 0 { t.Fatal("known peer addresses should contain 0 entry") } // peer should not be among live connections - if k.conns.Size() != 0 { + if tk.conns.Size() != 0 { t.Fatal("live peers should contain 0 entry") } } func TestSuggestPeerRetries(t *testing.T) { - k := newTestKademlia("00000000") - k.RetryInterval = int64(300 * time.Millisecond) // cycle - k.MaxRetries = 50 - k.RetryExponent = 2 + tk := newTestKademlia(t, "00000000") + tk.RetryInterval = int64(300 * time.Millisecond) // cycle + tk.MaxRetries = 50 + tk.RetryExponent = 2 sleep := func(n int) { - ts := k.RetryInterval + ts := tk.RetryInterval for i := 1; i < n; i++ { - ts *= int64(k.RetryExponent) + ts *= int64(tk.RetryExponent) } time.Sleep(time.Duration(ts)) } - Register(k, "01000000") - On(k, "00000001", "00000010") - err := testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.Register("01000000") + tk.On("00000001", "00000010") + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(1) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(1) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(2) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(2) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } - + tk.checkSuggestPeer("", 0, false) } func TestKademliaHiveString(t *testing.T) { - k := newTestKademlia("00000000") - On(k, "01000000", "00100000") - Register(k, "10000000", "10000001") - k.MaxProxDisplay = 8 - h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), NeighbourhoodSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + tk := newTestKademlia(t, "00000000") + tk.On("01000000", "00100000") + tk.Register("10000000", "10000001") + tk.MaxProxDisplay = 8 + h := tk.String() + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[104:] != h[104:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } } -// testKademliaCase constructs the kademlia and PeerPot map to validate -// the SuggestPeer and Healthy methods for provided hex-encoded addresses. -// Argument pivotAddr is the address of the kademlia. -func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { - addr := common.Hex2Bytes(pivotAddr) - var byteAddrs [][]byte - for _, ahex := range addrs { - byteAddrs = append(byteAddrs, common.Hex2Bytes(ahex)) - } - - k := NewKademlia(addr, NewKadParams()) - - // our pivot kademlia is the last one in the array - for _, a := range byteAddrs { - if bytes.Equal(a, addr) { - continue - } - p := &BzzAddr{OAddr: a, UAddr: a} - if err := k.Register(p); err != nil { - t.Fatalf("a %x addr %x: %v", a, addr, err) - } - } - - for { - a, _, _ := k.SuggestPeer() - if a == nil { - break - } - k.On(NewPeer(&BzzPeer{BzzAddr: a}, k)) - } - - assertHealth(t, k, true, true) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 12:18:24 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 7efef1 -population: 9 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 d7e5 ec56 | 18 ec56 (0) d7e5 (0) d9e0 (0) c735 (0) -001 2 18f1 3176 | 14 18f1 (0) 10bb (0) 10d1 (0) 0421 (0) -002 2 52aa 47cd | 11 52aa (0) 51d9 (0) 5161 (0) 5130 (0) -003 1 646e | 1 646e (0) -004 0 | 3 769c (0) 76d1 (0) 7656 (0) -============ DEPTH: 5 ========================================== -005 1 7a48 | 1 7a48 (0) -006 1 7cbd | 1 7cbd (0) -007 0 | 0 -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase1(t *testing.T) { - testKademliaCase(t, - "7efef1c41d77f843ad167be95f6660567eb8a4a59f39240000cce2e0d65baf8e", - "ec560e6a4806aa37f147ee83687f3cf044d9953e61eedb8c34b6d50d9e2c5623", - "646e9540c84f6a2f9cf6585d45a4c219573b4fd1b64a3c9a1386fc5cf98c0d4d", - "18f13c5fba653781019025ab10e8d2fdc916d6448729268afe9e928ffcdbb8e8", - "317617acf99b4ffddda8a736f8fc6c6ede0bf690bc23d834123823e6d03e2f69", - "d7e52d9647a5d1c27a68c3ee65d543be3947ae4b68537b236d71ef9cb15fb9ab", - "7a48f75f8ca60487ae42d6f92b785581b40b91f2da551ae73d5eae46640e02e8", - "7cbd42350bde8e18ae5b955b5450f8e2cef3419f92fbf5598160c60fd78619f0", - "52aa3ddec61f4d48dd505a2385403c634f6ad06ee1d99c5c90a5ba6006f9af9c", - "47cdb6fa93eeb8bc91a417ff4e3b14a9c2ea85137462e2f575fae97f0c4be60d", - "5161943eb42e2a03e715fe8afa1009ff5200060c870ead6ab103f63f26cb107f", - "a38eaa1255f76bf883ca0830c86e8c4bb7eed259a8348aae9b03f21f90105bee", - "b2522bdf1ab26f324e75424fdf6e493b47e8a27687fe76347607b344fc010075", - "5bd7213964efb2580b91d02ac31ef126838abeba342f5dbdbe8d4d03562671a2", - "0b531adb82744768b694d7f94f73d4f0c9de591266108daeb8c74066bfc9c9ca", - "28501f59f70e888d399570145ed884353e017443c675aa12731ada7c87ea14f7", - "4a45f1fc63e1a9cb9dfa44c98da2f3d20c2923e5d75ff60b2db9d1bdb0c54d51", - "b193431ee35cd32de95805e7c1c749450c47486595aae7195ea6b6019a64fd61", - "baebf36a1e35a7ed834e1c72faf44ba16c159fa47d3289ceb3ca35fefa8739b5", - "a3659bd32e05fa36c8d20dbaaed8362bf1a8a7bd116aed62d8a43a2efbdf513f", - "10d1b50881a4770ebebdd0a75589dabb931e6716747b0f65fd6b080b88c4fdb6", - "3c76b8ca5c7ce6a03320646826213f59229626bf5b9d25da0c3ec0662dcb8ff3", - "4d72a04ddeb851a68cd197ef9a92a3e2ff01fbbff638e64929dd1a9c2e150112", - "c7353d320987956075b5bc1668571c7a36c800d5598fdc4832ec6569561e15d1", - "d9e0c7c90878c20ab7639d5954756f54775404b3483407fe1b483635182734f6", - "8fca67216b7939c0824fb06c5279901a94da41da9482b000f56df9906736ee75", - "460719d7f7aa7d7438f0eaf30333484fa3bd0f233632c10ba89e6e46dd3604be", - "0421d92c8a1c79ed5d01305a3d25aaf22a8f5f9e3d4bc80da47ee16ce20465fe", - "3441d9d9c0f05820a1bb6459fc7d8ef266a1bd929e7db939a10f544efe8261ea", - "ab198a66c293586746758468c610e5d3914d4ce629147eff6dd55a31f863ff8f", - "3a1c8c16b0763f3d2c35269f454ff779d1255e954d2deaf6c040fb3f0bcdc945", - "5561c0ea3b203e173b11e6aa9d0e621a4e10b1d8b178b8fe375220806557b823", - "7656caccdc79cd8d7ce66d415cc96a718e8271c62fb35746bfc2b49faf3eebf3", - "5130594fd54c1652cf2debde2c4204573ed76555d1e26757fe345b409af1544a", - "76d1e83c71ca246d042e37ff1db181f2776265fbcfdc890ce230bfa617c9c2f0", - "89580231962624c53968c1b0095b4a2732b2a2640a19fdd7d21fd064fcc0a5ef", - "3d10d001fff44680c7417dd66ecf2e984f0baa20a9bbcea348583ba5ff210c4f", - "43754e323f0f3a1155b1852bd6edd55da86b8c4cfe3df8b33733fca50fc202b8", - "a9e7b1bb763ae6452ddcacd174993f82977d81a85206bb2ae3c842e2d8e19b4c", - "10bb07da7bc7c7757f74149eff167d528a94a253cdc694a863f4d50054c00b6d", - "28f0bc1b44658548d6e05dd16d4c2fe77f1da5d48b6774bc4263b045725d0c19", - "835fbbf1d16ba7347b6e2fc552d6e982148d29c624ea20383850df3c810fa8fc", - "8e236c56a77d7f46e41e80f7092b1a68cd8e92f6156365f41813ad1ca2c6b6f3", - "51d9c857e9238c49186e37b4eccf17a82de3d5739f026f6043798ab531456e73", - "bbddf7db6a682225301f36a9fd5b0d0121d2951753e1681295f3465352ad511f", - "2690a910c33ee37b91eb6c4e0731d1d345e2dc3b46d308503a6e85bbc242c69e", - "769ce86aa90b518b7ed382f9fdacfbed93574e18dc98fe6c342e4f9f409c2d5a", - "ba3bebec689ce51d3e12776c45f80d25164fdfb694a8122d908081aaa2e7122c", - "3a51f4146ea90a815d0d283d1ceb20b928d8b4d45875e892696986a3c0d8fb9b", - "81968a2d8fb39114342ee1da85254ec51e0608d7f0f6997c2a8354c260a71009", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 18:43:48 UTC 2018 KΛÐΞMLIΛ hive: queen's address: bc7f3b -population: 9 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 0f49 67ff | 28 0f49 (0) 0211 (0) 07b2 (0) 0703 (0) -001 2 e84b f3a4 | 13 f3a4 (0) e84b (0) e58b (0) e60b (0) -002 1 8dba | 1 8dba (0) -003 2 a008 ad72 | 2 ad72 (0) a008 (0) -004 0 | 3 b61f (0) b27f (0) b027 (0) -============ DEPTH: 5 ========================================== -005 1 ba19 | 1 ba19 (0) -006 0 | 0 -007 1 bdd6 | 1 bdd6 (0) -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase2(t *testing.T) { - testKademliaCase(t, - "bc7f3b6a4a7e3c91b100ca6680b6c06ff407972b88956324ca853295893e0237", "67ffb61d3aa27449d277016188f35f19e2321fbda5008c68cf6303faa080534f", "600cd54c842eadac1729c04abfc369bc244572ca76117105b9dd910283b82730", "d955a05409650de151218557425105a8aa2867bb6a0e0462fa1cf90abcf87ad6", "7a6b726de45abdf7bb3e5fd9fb0dc8932270ca4dedef92238c80c05bcdb570e3", "263e99424ebfdb652adb4e3dcd27d59e11bb7ae1c057b3ef6f390d0228006254", "ba195d1a53aafde68e661c64d39db8c2a73505bf336125c15c3560de3b48b7ed", "3458c762169937115f67cabc35a6c384ed70293a8aec37b077a6c1b8e02d510e", "4ef4dc2e28ac6efdba57e134ac24dd4e0be68b9d54f7006515eb9509105f700c", "2a8782b79b0c24b9714dfd2c8ff1932bebc08aa6520b4eaeaa59ff781238890c", "625d02e960506f4524e9cdeac85b33faf3ea437fceadbd478b62b78720cf24fc", "e051a36a8c8637f520ba259c9ed3fadaf740dadc6a04c3f0e21778ebd4cd6ac4", "e34bc014fa2504f707bb3d904872b56c2fa250bee3cb19a147a0418541f1bd90", "28036dc79add95799916893890add5d8972f3b95325a509d6ded3d448f4dc652", "1b013c407794fa2e4c955d8f51cbc6bd78588a174b6548246b291281304b5409", "34f71b68698e1534095ff23ee9c35bf64c7f12b8463e7c6f6b19c25cf03928b4", "c712c6e9bbb7076832972a95890e340b94ed735935c3c0bb788e61f011b59479", "a008d5becdcda4b9dbfdaafc3cec586cf61dcf2d4b713b6168fff02e3b9f0b08", "29de15555cdbebaab214009e416ee92f947dcec5dab9894129f50f1b17138f34", "5df9449f700bd4b5a23688b68b293f2e92fa6ca524c93bc6bb9936efba9d9ada", "3ab0168a5f87fedc6a39b53c628256ac87a98670d8691bbdaaecec22418d13a2", "1ee299b2d2a74a568494130e6869e66d57982d345c482a0e0eeb285ac219ae3b", "e0e0e3b860cea9b7a74cf1b0675cc632dc64e80a02f20bbc5e96e2e8bb670606", "dc1ba6f169b0fcdcca021dcebaf39fe5d4875e7e69b854fad65687c1d7719ec0", "d321f73e42fcfb1d3a303eddf018ca5dffdcfd5567cd5ec1212f045f6a07e47d", "070320c3da7b542e5ca8aaf6a0a53d2bb5113ed264ab1db2dceee17c729edcb1", "17d314d65fdd136b50d182d2c8f5edf16e7838c2be8cf2c00abe4b406dbcd1d8", "e60b99e0a06f7d2d99d84085f67cdf8cc22a9ae22c339365d80f90289834a2b4", "02115771e18932e1f67a45f11f5bf743c5dae97fbc477d34d35c996012420eac", "3102a40eb2e5060353dd19bf61eeec8782dd1bebfcb57f4c796912252b591827", "8dbaf231062f2dc7ddaba5f9c7761b0c21292be51bf8c2ef503f31d4a2f63f79", "b02787b713c83a9f9183216310f04251994e04c2763a9024731562e8978e7cc4", "b27fe6cd33989e10909ce794c4b0b88feae286b614a59d49a3444c1a7b51ea82", "07b2d2c94fdc6fd148fe23be2ed9eff54f5e12548f29ed8416e6860fc894466f", "e58bf9f451ef62ac44ff0a9bb0610ec0fd14d423235954f0d3695e83017cbfc4", "bdd600b91bb79d1ee0053b854de308cfaa7e2abce575ea6815a0a7b3449609c2", "0f49c93c1edc7999920b21977cedd51a763940dac32e319feb9c1df2da0f3071", "7cbf0297cd41acf655cd6f960d7aaf61479edb4189d5c001cbc730861f0deb41", "79265193778d87ad626a5f59397bc075872d7302a12634ce2451a767d0a82da2", "2fe7d705f7c370b9243dbaafe007d555ff58d218822fca49d347b12a0282457c", "e84bc0c83d05e55a0080eed41dda5a795da4b9313a4da697142e69a65834cbb3", "cc4d278bd9aa0e9fb3cd8d2e0d68fb791aab5de4b120b845c409effbed47a180", "1a2317a8646cd4b6d3c4aa4cc25f676533abb689cf180787db216880a1239ad8", "cbafd6568cf8e99076208e6b6843f5808a7087897c67aad0c54694669398f889", "7b7c8357255fc37b4dae0e1af61589035fd39ff627e0938c6b3da8b4e4ec5d23", "2b8d782c1f5bac46c922cf439f6aa79f91e9ba5ffc0020d58455188a2075b334", "b61f45af2306705740742e76197a119235584ced01ef3f7cf3d4370f6c557cd1", "2775612e7cdae2780bf494c370bdcbe69c55e4a1363b1dc79ea0135e61221cce", "f3a49bb22f40885e961299abfa697a7df690a79f067bf3a4847a3ad48d826c9f", "ad724ac218dc133c0aadf4618eae21fdd0c2f3787af279846b49e2b4f97ff167", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 19:04:35 UTC 2018 KΛÐΞMLIΛ hive: queen's address: b4822e -population: 8 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 786c 774b | 29 774b (0) 786c (0) 7a79 (0) 7d2f (0) -001 2 d9de cf19 | 10 cf19 (0) d9de (0) d2ff (0) d2a2 (0) -002 2 8ca1 8d74 | 5 8d74 (0) 8ca1 (0) 9793 (0) 9f51 (0) -003 0 | 0 -004 0 | 3 bfac (0) bcbb (0) bde9 (0) -005 0 | 0 -============ DEPTH: 6 ========================================== -006 1 b660 | 1 b660 (0) -007 0 | 0 -008 1 b450 | 1 b450 (0) -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase3(t *testing.T) { - testKademliaCase(t, - "b4822e874a01b94ac3a35c821e6db131e785c2fcbb3556e84b36102caf09b091", "2ecf54ea38d58f9cfc3862e54e5854a7c506fbc640e0b38e46d7d45a19794999", "442374092be50fc7392e8dd3f6fab3158ff7f14f26ff98060aed9b2eecf0b97d", "b450a4a67fcfa3b976cf023d8f1f15052b727f712198ce901630efe2f95db191", "9a7291638eb1c989a6dd6661a42c735b23ac6605b5d3e428aa5ffe650e892c85", "67f62eeab9804cfcac02b25ebeab9113d1b9d03dd5200b1c5a324cc0163e722f", "2e4a0e4b53bca4a9d7e2734150e9f579f29a255ade18a268461b20d026c9ee90", "30dd79c5fcdaa1b106f6960c45c9fde7c046aa3d931088d98c52ab759d0b2ac4", "97936fb5a581e59753c54fa5feec493714f2218245f61f97a62eafd4699433e4", "3a2899b6e129e3e193f6e2aefb82589c948c246d2ec1d4272af32ef3b2660f44", "f0e2a8aa88e67269e9952431ef12e5b29b7f41a1871fbfc38567fad95655d607", "7fa12b3f3c5f8383bfc644b958f72a486969733fa097d8952b3eb4f7b4f73192", "360c167aad5fc992656d6010ec45fdce5bcd492ad9608bc515e2be70d4e430c1", "fe21bc969b3d8e5a64a6484a829c1e04208f26f3cd4de6afcbc172a5bd17f1f1", "b660a1f40141d7ccd282fe5bd9838744119bd1cb3780498b5173578cc5ad308f", "44dcb3370e76680e2fba8cd986ad45ff0b77ca45680ee8d950e47922c4af6226", "8ca126923d17fccb689647307b89f38aa14e2a7b9ebcf3c1e31ccf3d2291a3bc", "f0ae19ae9ce6329327cbf42baf090e084c196b0877d8c7b69997e0123be23ef8", "d2a2a217385158e3e1e348883a14bc423e57daa12077e8c49797d16121ea0810", "f5467ccd85bb4ebe768527db520a210459969a5f1fae6e07b43f519799f0b224", "68be5fd9f9d142a5099e3609011fe3bab7bb992c595999e31e0b3d1668dfb3cf", "4d49a8a476e4934afc6b5c36db9bece3ed1804f20b952da5a21b2b0de766aa73", "ea7155745ef3fb2d099513887a2ba279333ced65c65facbd890ce58bd3fce772", "cf19f51f4e848053d289ac95a9138cdd23fc3077ae913cd58cda8cc7a521b2e1", "590b1cd41c7e6144e76b5cd515a3a4d0a4317624620a3f1685f43ae68bdcd890", "d2ffe0626b5f94a7e00fa0b506e7455a3d9399c15800db108d5e715ef5f6e346", "69630878c50a91f6c2edd23a706bfa0b50bd5661672a37d67bab38e6bca3b698", "445e9067079899bb5faafaca915ae6c0f6b1b730a5a628835dd827636f7feb1e", "6461c77491f1c4825958949f23c153e6e1759a5be53abbcee17c9da3867f3141", "23a235f4083771ccc207771daceda700b525a59ab586788d4f6892e69e34a6e2", "bde99f79ef41a81607ddcf92b9f95dcbc6c3537e91e8bf740e193dc73b19485e", "177957c0e5f0fbd12b88022a91768095d193830986caec8d888097d3ff4310b8", "bcbbdbaa4cdf8352422072f332e05111b732354a35c4d7c617ce1fc3b8b42a5a", "774b6717fdfb0d1629fb9d4c04a9ca40079ae2955d7f82e897477055ed017abb", "16443bf625be6d39ecaa6f114e5d2c1d47a64bfd3c13808d94b55b6b6acef2ee", "8d7495d9008066505ed00ce8198af82bfa5a6b4c08768b4c9fb3aa4eb0b0cca2", "15800849a53349508cb382959527f6c3cf1a46158ff1e6e2316b7dea7967e35f", "7a792f0f4a2b731781d1b244b2a57947f1a2e32900a1c0793449f9f7ae18a7b7", "5e517c2832c9deaa7df77c7bad4d20fd6eda2b7815e155e68bc48238fac1416f", "9f51a14f0019c72bd1d472706d8c80a18c1873c6a0663e754b60eae8094483d7", "7d2fabb565122521d22ba99fed9e5be6a458fbc93156d54db27d97a00b8c3a97", "786c9e412a7db4ec278891fa534caa9a1d1a028c631c6f3aeb9c4d96ad895c36", "3bd6341d40641c2632a5a0cd7a63553a04e251efd7195897a1d27e02a7a8bfde", "31efd1f5fb57b8cff0318d77a1a9e8d67e1d1c8d18ce90f99c3a240dff48cdc8", "d9de3e1156ce1380150948acbcfecd99c96e7f4b0bc97745f4681593d017f74f", "427a2201e09f9583cd990c03b81b58148c297d474a3b50f498d83b1c7a9414cd", "bfaca11596d3dec406a9fcf5d97536516dfe7f0e3b12078428a7e1700e25218a", "351c4770a097248a650008152d0cab5825d048bef770da7f3364f59d1e721bc0", "ee00f205d1486b2be7381d962bd2867263758e880529e4e2bfedfa613bbc0e71", "6aa3b6418d89e3348e4859c823ef4d6d7cd46aa7f7e77aba586c4214d760d8f8", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 19:16:25 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 9a90fe -population: 8 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 72ef 4e6c | 24 0b1e (0) 0d66 (0) 17f5 (0) 17e8 (0) -001 2 fc2b fa47 | 13 fa47 (0) fc2b (0) fffd (0) ecef (0) -002 2 b847 afa8 | 6 afa8 (0) ad77 (0) bb7c (0) b847 (0) -003 0 | 0 -004 0 | 4 91fc (0) 957d (0) 9482 (0) 949a (0) -============ DEPTH: 5 ========================================== -005 1 9ccf | 1 9ccf (0) -006 0 | 0 -007 1 9bb2 | 1 9bb2 (0) -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase4(t *testing.T) { - testKademliaCase(t, - "9a90fe3506277244549064b8c3276abb06284a199d9063a97331947f2b7da7f4", - "c19359eddef24b7be1a833b4475f212cd944263627a53f9ef4837d106c247730", "fc2b6fef99ef947f7e57c3df376891769e2a2fd83d2b8e634e0fc1e91eaa080c", "ecefc0e1a8ea7bb4b48c469e077401fce175dd75294255b96c4e54f6a2950a55", "bb7ce598efc056bba343cc2614aa3f67a575557561290b44c73a63f8f433f9f7", "55fbee6ca52dfd7f0be0db969ee8e524b654ab4f0cce7c05d83887d7d2a15460", "afa852b6b319998c6a283cc0c82d2f5b8e9410075d7700f3012761f1cfbd0f76", "36c370cfb63f2087971ba6e58d7585b04e16b8f0da335efb91554c2dd8fe191c", "6be41e029985edebc901fb77fc4fb65516b6d85086e2a98bfa3159c99391e585", "dd3cfc72ea553e7d2b28f0037a65646b30955b929d29ba4c40f4a2a811248e77", "da3a8f18e09c7b0ca235c4e33e1441a5188f1df023138bf207753ee63e768f7d", "de9e3ab4dc572d54a2d4b878329fd832bb51a149f4ce167316eeb177b61e7e01", "4e6c1ecde6ed917706257fe020a1d02d2e9d87fca4c85f0f7b132491008c5032", "72ef04b77a070e13463b3529dd312bcacfb7a12d20dc597f5ec3de0501e9b834", "3fef57186675d524ab8bb1f54ba8cb68610babca1247c0c46dbb60aed003c69d", "1d8e6b71f7a052865d6558d4ba44ad5fab7b908cc1badf5766822e1c20d0d823", "6be2f2b4ffa173014d4ec7df157d289744a2bda54bb876b264ccfa898a0da315", "b0ba3fff8643f9985c744327b0c4c869763509fd5da2de9a80a4a0a082021255", "9ccf40b9406ba2e6567101fb9b4e5334a9ec74263eff47267da266ba45e6c158", "d7347f02c180a448e60f73931845062ce00048750b584790278e9c93ef31ad81", "b68c6359a22b3bee6fecb8804311cfd816648ea31d530c9fb48e477e029d707a", "0d668a18ad7c2820214df6df95a6c855ce19fb1cb765f8ca620e45db76686d37", "3fbd2663bff65533246f1fabb9f38086854c6218aeb3dc9ac6ac73d4f0988f91", "949aa5719ca846052bfaa1b38c97b6eca3df3e24c0e0630042c6bccafbb4cdb5", "77b8a2b917bef5d54f3792183b014cca7798f713ff14fe0b2ac79b4c9f6f996d", "17e853cbd8dc00cba3cd9ffeb36f26a9f41a0eb92f80b62c2cda16771c935388", "5f682ed7a8cf2f98387c3def7c97f9f05ae39e39d393eeca3cf621268d6347f8", "ad77487eaf11fd8084ba4517a51766eb0e5b77dd3492dfa79aa3a2802fb29d20", "d247cfcacf9a8200ebaddf639f8c926ab0a001abe682f40df3785e80ed124e91", "195589442e11907eede1ee6524157f1125f68399f3170c835ff81c603b069f6c", "5b5ca0a67f3c54e7d3a6a862ef56168ec9ed1f4945e6c24de6d336b2be2e6f8c", "56430e4caa253015f1f998dce4a48a88af1953f68e94eca14f53074ae9c3e467", "0b1eed6a5bf612d1d8e08f5c546f3d12e838568fd3aa43ed4c537f10c65545d6", "7058db19a56dfff01988ac4a62e1310597f9c8d7ebde6890dadabf047d722d39", "b847380d6888ff7cd11402d086b19eccc40950b52c9d67e73cb4f8462f5df078", "df6c048419a2290ab546d527e9eeba349e7f7e1759bafe4adac507ce60ef9670", "91fc5b4b24fc3fbfea7f9a3d0f0437cb5733c0c2345d8bdffd7048d6e3b8a37b", "957d8ea51b37523952b6f5ae95462fcd4aed1483ef32cc80b69580aaeee03606", "efa82e4e91ad9ab781977400e9ac0bb9de7389aaedebdae979b73d1d3b8d72b0", "7400c9f3f3fc0cc6fe8cc37ab24b9771f44e9f78be913f73cd35fc4be030d6bd", "9bb28f4122d61f7bb56fe27ef706159fb802fef0f5de9dfa32c9c5b3183235f1", "40a8de6e98953498b806614532ea4abf8b99ad7f9719fb68203a6eae2efa5b2a", "412de0b218b8f7dcacc9205cd16ffb4eca5b838f46a2f4f9f534026061a47308", "17f56ecad51075080680ad9faa0fd8946b824d3296ddb20be07f9809fe8d1c5a", "fffd4e7ae885a41948a342b6647955a7ec8a8039039f510cff467ef597675457", "35e78e11b5ac46a29dd04ab0043136c3291f4ca56cb949ace33111ed56395463", "94824fc80230af82077c83bfc01dc9675b1f9d3d538b1e5f41c21ac753598691", "fa470ae314ca3fce493f21b423eef2a49522e09126f6f2326fa3c9cac0b344f7", "7078860b5b621b21ac7b95f9fc4739c8235ce5066a8b9bd7d938146a34fa88ec", "eea53560f0428bfd2eca4f86a5ce9dec5ff1309129a975d73465c1c9e9da71d1", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 19:25:18 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 5dd5c7 -population: 13 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 e528 fad0 | 22 fad0 (0) e528 (0) e3bb (0) ed13 (0) -001 3 3f30 18e0 1dd3 | 7 3f30 (0) 23db (0) 10b6 (0) 18e0 (0) -002 4 7c54 7804 61e4 60f9 | 10 61e4 (0) 60f9 (0) 636c (0) 7186 (0) -003 2 40ae 4bae | 5 4bae (0) 4d5c (0) 403a (0) 40ae (0) -004 0 | 0 -005 0 | 3 5808 (0) 5a0e (0) 5bdb (0) -============ DEPTH: 6 ========================================== -006 2 5f14 5f61 | 2 5f14 (0) 5f61 (0) -007 0 | 0 -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase5(t *testing.T) { - testKademliaCase(t, - "5dd5c77dd9006a800478fcebb02d48d4036389e7d3c8f6a83b97dbad13f4c0a9", - "78fafa0809929a1279ece089a51d12457c2d8416dff859aeb2ccc24bb50df5ec", "1dd39b1257e745f147cbbc3cadd609ccd6207c41056dbc4254bba5d2527d3ee5", "5f61dd66d4d94aec8fcc3ce0e7885c7edf30c43143fa730e2841c5d28e3cd081", "8aa8b0472cb351d967e575ad05c4b9f393e76c4b01ef4b3a54aac5283b78abc9", "4502f385152a915b438a6726ce3ea9342e7a6db91a23c2f6bee83a885ed7eb82", "718677a504249db47525e959ef1784bed167e1c46f1e0275b9c7b588e28a3758", "7c54c6ed1f8376323896ed3a4e048866410de189e9599dd89bf312ca4adb96b5", "18e03bd3378126c09e799a497150da5c24c895aedc84b6f0dbae41fc4bac081a", "23db76ac9e6e58d9f5395ca78252513a7b4118b4155f8462d3d5eec62486cadc", "40ae0e8f065e96c7adb7fa39505136401f01780481e678d718b7f6dbb2c906ec", "c1539998b8bae19d339d6bbb691f4e9daeb0e86847545229e80fe0dffe716e92", "ed139d73a2699e205574c08722ca9f030ad2d866c662f1112a276b91421c3cb9", "5bdb19584b7a36d09ca689422ef7e6bb681b8f2558a6b2177a8f7c812f631022", "636c9de7fe234ffc15d67a504c69702c719f626c17461d3f2918e924cd9d69e2", "de4455413ff9335c440d52458c6544191bd58a16d85f700c1de53b62773064ea", "de1963310849527acabc7885b6e345a56406a8f23e35e436b6d9725e69a79a83", "a80a50a467f561210a114cba6c7fb1489ed43a14d61a9edd70e2eb15c31f074d", "7804f12b8d8e6e4b375b242058242068a3809385e05df0e64973cde805cf729c", "60f9aa320c02c6f2e6370aa740cf7cea38083fa95fca8c99552cda52935c1520", "d8da963602390f6c002c00ce62a84b514edfce9ebde035b277a957264bb54d21", "8463d93256e026fe436abad44697152b9a56ac8e06a0583d318e9571b83d073c", "9a3f78fcefb9a05e40a23de55f6153d7a8b9d973ede43a380bf46bb3b3847de1", "e3bb576f4b3760b9ca6bff59326f4ebfc4a669d263fb7d67ab9797adea54ed13", "4d5cdbd6dcca5bdf819a0fe8d175dc55cc96f088d37462acd5ea14bc6296bdbe", "5a0ed28de7b5258c727cb85447071c74c00a5fbba9e6bc0393bc51944d04ab2a", "61e4ddb479c283c638f4edec24353b6cc7a3a13b930824aad016b0996ca93c47", "7e3610868acf714836cafaaa7b8c009a9ac6e3a6d443e5586cf661530a204ee2", "d74b244d4345d2c86e30a097105e4fb133d53c578320285132a952cdaa64416e", "cfeed57d0f935bfab89e3f630a7c97e0b1605f0724d85a008bbfb92cb47863a8", "580837af95055670e20d494978f60c7f1458dc4b9e389fc7aa4982b2aca3bce3", "df55c0c49e6c8a83d82dfa1c307d3bf6a20e18721c80d8ec4f1f68dc0a137ced", "5f149c51ce581ba32a285439a806c063ced01ccd4211cd024e6a615b8f216f95", "1eb76b00aeb127b10dd1b7cd4c3edeb4d812b5a658f0feb13e85c4d2b7c6fe06", "7a56ba7c3fb7cbfb5561a46a75d95d7722096b45771ec16e6fa7bbfab0b35dfe", "4bae85ad88c28470f0015246d530adc0cd1778bdd5145c3c6b538ee50c4e04bd", "afd1892e2a7145c99ec0ebe9ded0d3fec21089b277a68d47f45961ec5e39e7e0", "953138885d7b36b0ef79e46030f8e61fd7037fbe5ce9e0a94d728e8c8d7eab86", "de761613ef305e4f628cb6bf97d7b7dc69a9d513dc233630792de97bcda777a6", "3f3087280063d09504c084bbf7fdf984347a72b50d097fd5b086ffabb5b3fb4c", "7d18a94bb1ebfdef4d3e454d2db8cb772f30ca57920dd1e402184a9e598581a0", "a7d6fbdc9126d9f10d10617f49fb9f5474ffe1b229f76b7dd27cebba30eccb5d", "fad0246303618353d1387ec10c09ee991eb6180697ed3470ed9a6b377695203d", "1cf66e09ea51ee5c23df26615a9e7420be2ac8063f28f60a3bc86020e94fe6f3", "8269cdaa153da7c358b0b940791af74d7c651cd4d3f5ed13acfe6d0f2c539e7f", "90d52eaaa60e74bf1c79106113f2599471a902d7b1c39ac1f55b20604f453c09", "9788fd0c09190a3f3d0541f68073a2f44c2fcc45bb97558a7c319f36c25a75b3", "10b68fc44157ecfdae238ee6c1ce0333f906ad04d1a4cb1505c8e35c3c87fbb0", "e5284117fdf3757920475c786e0004cb00ba0932163659a89b36651a01e57394", "403ad51d911e113dcd5f9ff58c94f6d278886a2a4da64c3ceca2083282c92de3", - ) -} - func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { rw := &p2p.MsgPipeRW{} p := p2p.NewPeer(enode.ID{}, "foo", []p2p.Cap{}) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index e13517a63eaf..0abb214db854 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -270,7 +270,7 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } - log.Info(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive)) + log.Debug(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive)) return healthy.KnowNN && healthy.ConnectNN, nil } From 94cd49d17d6c854442a6638e1e16872e5b4cafb2 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 15 Jan 2019 08:18:59 +0100 Subject: [PATCH 7/8] swarm/network: kademlia suggestPeer cleanup, improved comments --- swarm/network/kademlia.go | 80 ++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index a3c5482ea397..4ea045819e09 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -173,30 +173,30 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c k.lock.Lock() defer k.lock.Unlock() radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) - // find the undersaturated bins from lowest cardinality from shallow to deep - // first map unsaturated bins to cardinalities - spo := make(map[int][]int) - var prev int - saturationDepth = -1 - var pastDepth bool + // first collect undersaturated bins starting from smallest size from shallow to deep + // saturation is a map: bin size (int) -> deepening list of PO bins with that size ([]int) + saturation := make(map[int][]int) + var lastPO int // the last non-empty PO bin in the iteration + saturationDepth = -1 // the deepest PO such that all shallower bins have >= k.MinBinSize peers + var pastDepth bool // whether po of iteration >= depth k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { // process skipped empty bins - for ; prev < po; prev++ { + for ; lastPO < po; lastPO++ { // find the lowest unsaturated bin if saturationDepth == -1 { - saturationDepth = prev + saturationDepth = lastPO } // if there is an empty bin, depth is surely passed pastDepth = true - spo[0] = append(spo[0], prev) + saturation[0] = append(saturation[0], lastPO) } - prev = po + 1 + lastPO = po + 1 // past radius, depth is surely passed if po >= radius { pastDepth = true } - // beyond depth the bin is treated as unsaturated to achieve full connectivity - // to all neighbours + // beyond depth the bin is treated as unsaturated even if size >= k.MinBinSize + // in order to achieve full connectivity to all neighbours if pastDepth && size >= k.MinBinSize { size = k.MinBinSize - 1 } @@ -206,7 +206,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c if saturationDepth == -1 { saturationDepth = po } - spo[size] = append(spo[size], po) + saturation[size] = append(saturation[size], po) } return true }) @@ -217,53 +217,57 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c nearestAddrAt = po return false }) - for ; prev <= nearestAddrAt; prev++ { - spo[0] = append(spo[0], prev) + // including bins as size 0 has the effect that requesting connection + // is prioritised over non-empty shallower bins + for ; lastPO <= nearestAddrAt; lastPO++ { + saturation[0] = append(saturation[0], lastPO) } // all PO bins are saturated, ie., minsize >= k.MinBinSize, no peer suggested - if len(spo) == 0 { + if len(saturation) == 0 { return nil, 0, false } - // find the first callable peer - // starting from the lowest cardinality + // find the first callable peer in the address book + // starting from the bins with smallest size proceeding from shallow to deep // for each bin (up until neighbourhood radius) we find callable candidate peers - for i := 0; i < k.MinBinSize && suggestedPeer == nil; i++ { - bins, ok := spo[i] + for size := 0; size < k.MinBinSize && suggestedPeer == nil; size++ { + bins, ok := saturation[size] if !ok { + // no bin with this size continue } - ci := 0 - cpo := bins[0] - k.addrs.EachBin(k.base, Pof, cpo, func(po, _ int, f func(func(pot.Val) bool) bool) bool { - cpo = bins[ci] - // find the next bin with cardinality i - if cpo == po { - ci++ + cur := 0 + curPO := bins[0] + k.addrs.EachBin(k.base, Pof, curPO, func(po, _ int, f func(func(pot.Val) bool) bool) bool { + curPO = bins[cur] + // find the next bin that has size size + if curPO == po { + cur++ } else { // skip bins that have no addresses - for ; ci < len(bins) && cpo < po; ci++ { - cpo = bins[ci] + for ; cur < len(bins) && curPO < po; cur++ { + curPO = bins[cur] } - if po < cpo { - ci-- + if po < curPO { + cur-- return true } // stop if there are no addresses - if cpo < po { + if curPO < po { return false } } + // curPO found // find a callable peer out of the addresses in the unsaturated bin // stop if found f(func(val pot.Val) bool { e := val.(*entry) - c := k.callable(e) - if c { + if k.callable(e) { suggestedPeer = e.BzzAddr + return false } - return !c + return true }) - return ci < len(bins) && suggestedPeer == nil + return cur < len(bins) && suggestedPeer == nil }) } @@ -274,7 +278,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c return suggestedPeer, 0, false } -// On inserts the peer as a kademlia peer into the live peers +// On inserts the peer as a kademlia peer into the live peers func (k *Kademlia) On(p *Peer) (uint8, bool) { k.lock.Lock() defer k.lock.Unlock() @@ -434,7 +438,7 @@ func (k *Kademlia) NeighbourhoodDepth() (depth int) { } // neighbourhoodRadiusForPot returns the neighbourhood radius of the kademlia -// neighbourhood radius encloses the nearest neighbour set with cardinality >= neighbourhoodSize +// neighbourhood radius encloses the nearest neighbour set with size >= neighbourhoodSize // i.e., neighbourhood radius is the deepest PO such that all bins not shallower altogether // contain at least neighbourhoodSize connected peers // if there is altogether less than neighbourhoodSize peers connected, it returns 0 From 80eca895a2ad28146ba309a36c66294734f94d3a Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 16 Jan 2019 09:45:41 +0100 Subject: [PATCH 8/8] swarm/network: minor comment, discovery test default arg --- swarm/network/kademlia.go | 5 +++-- swarm/network/simulations/discovery/discovery_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 4ea045819e09..f9b38fc48d07 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -173,8 +173,9 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c k.lock.Lock() defer k.lock.Unlock() radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) - // first collect undersaturated bins starting from smallest size from shallow to deep - // saturation is a map: bin size (int) -> deepening list of PO bins with that size ([]int) + // collect undersaturated bins in ascending order of number of connected peers + // and from shallow to deep (ascending order of PO) + // insert them in a map of bin arrays, keyed with the number of connected peers saturation := make(map[int][]int) var lastPO int // the last non-empty PO bin in the iteration saturationDepth = -1 // the deepest PO such that all shallower bins have >= k.MinBinSize peers diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 0abb214db854..e695bc4ac054 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -82,7 +82,7 @@ func getDbStore(nodeID string) (*state.DBStore, error) { } var ( - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + nodeCount = flag.Int("nodes", 32, "number of nodes to create (default 32)") initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") loglevel = flag.Int("loglevel", 3, "verbosity of logs") rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs")