Skip to content

Commit

Permalink
Merge "[FAB-2576] Fix LE and gossip integration test"
Browse files Browse the repository at this point in the history
  • Loading branch information
@yacovm
yacovm authored and Gerrit Code Review committed Mar 4, 2017
2 parents eb94abc + e7b3c7a commit 2724ba4
Showing 1 changed file with 82 additions and 53 deletions.
135 changes: 82 additions & 53 deletions gossip/service/gossip_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/hyperledger/fabric/msp/mgmt/testtools"
"github.com/hyperledger/fabric/peer/gossip/mcs"
"github.com/hyperledger/fabric/protos/common"
"github.com/op/go-logging"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
Expand Down Expand Up @@ -105,7 +106,7 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) {
}
addPeersToChannel(t, n, 10000, channelName, gossips, peerIndexes)

waitForFullMembership(t, gossips, n, time.Second*30, time.Second*2)
waitForFullMembership(t, gossips, n, time.Second*20, time.Second*2)

services := make([]*electionService, n)

Expand All @@ -119,20 +120,26 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) {
deliverServiceFactory.service.running[channelName] = false

gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"})
gossips[i].(*gossipServiceImpl).chains[channelName].Stop()
service, exist := gossips[i].(*gossipServiceImpl).leaderElection[channelName]
assert.True(t, exist, "Leader election service should be created for peer %d and channel %s", i, channelName)
services[i] = &electionService{nil, false, 0}
services[i].LeaderElectionService = service
}

assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected")

assert.True(t, gossips[0].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName], "Delivery service should be started in peer %d", 0)
// Is single leader was elected.
assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected")

for i := 1; i < n; i++ {
assert.False(t, gossips[i].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName], "Delivery service should not be started in peer %d", i)
startsNum := 0
for i := 0; i < n; i++ {
// Is mockDeliverService.StartDeliverForChannel in current peer for the specific channel was invoked
if gossips[i].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName] {
startsNum++
}
}

assert.Equal(t, 1, startsNum, "Only for one peer delivery client should start")

stopPeers(gossips)
}

Expand Down Expand Up @@ -170,6 +177,7 @@ func TestWithStaticDeliverClientLeader(t *testing.T) {
gossips[i].(*gossipServiceImpl).deliveryFactory = deliverServiceFactory
deliverServiceFactory.service.running[channelName] = false
gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"})
gossips[i].(*gossipServiceImpl).chains[channelName].Stop()
}

for i := 0; i < n; i++ {
Expand Down Expand Up @@ -218,6 +226,7 @@ func TestWithStaticDeliverClientNotLeader(t *testing.T) {
gossips[i].(*gossipServiceImpl).deliveryFactory = deliverServiceFactory
deliverServiceFactory.service.running[channelName] = false
gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"})
gossips[i].(*gossipServiceImpl).chains[channelName].Stop()
}

for i := 0; i < n; i++ {
Expand Down Expand Up @@ -302,15 +311,14 @@ func (li *mockLedgerInfo) Commit(block *common.Block) error {

// Gets blocks with sequence numbers provided in the slice
func (li *mockLedgerInfo) GetBlocks(blockSeqs []uint64) []*common.Block {
return nil
return make([]*common.Block, 0)
}

// Closes committing service
func (li *mockLedgerInfo) Close() {
}

func TestLeaderElectionWithRealGossip(t *testing.T) {
t.Skip()

// Spawn 10 gossip instances with single channel and inside same organization
// Run leader election on top of each gossip instance and check that only one leader chosen
Expand All @@ -319,6 +327,9 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {
// Check correct leader still exist for first channel and new correct leader chosen in second channel
// Stop gossip instances of leader peers for both channels and see that new leader chosen for both

logging.SetLevel(logging.DEBUG, util.LoggingElectionModule)
logging.SetLevel(logging.DEBUG, util.LoggingServiceModule)

// Creating gossip service instances for peers
n := 10
gossips := startPeers(t, n, 10000)
Expand All @@ -335,7 +346,7 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {

logger.Warning("Starting leader election services")

//Stariting leader election services
//Starting leader election services
services := make([]*electionService, n)

for i := 0; i < n; i++ {
Expand All @@ -345,13 +356,17 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {

logger.Warning("Waiting for leader election")

assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected")
assert.True(t, services[0].callbackInvokeRes, "Callback func for peer 0 should be called (chanA)")
assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected")

for i := 1; i < n; i++ {
assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i)
assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i)
startsNum := 0
for i := 0; i < n; i++ {
// Is callback function was invoked by this leader election service instance
if services[i].callbackInvokeRes {
startsNum++
}
}
//Only leader should invoke callback function, so it is double check that only one leader exists
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanA")

// Adding some peers to new channel and creating leader election services for peers in new channel
// Expecting peer 1 (first in list of election services) to become leader of second channel
Expand All @@ -365,39 +380,52 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {
secondChannelServices[idx].LeaderElectionService = gossips[i].(*gossipServiceImpl).newLeaderElectionComponent(gossipCommon.ChainID(secondChannelName), secondChannelServices[idx].callback)
}

assert.True(t, waitForLeaderElection(t, secondChannelServices, 0, time.Second*30, time.Second*2), "One leader (peer 1) should be selected")
assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected")
assert.True(t, waitForLeaderElection(t, secondChannelServices, time.Second*30, time.Second*2), "One leader should be selected for chanB")
assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected for chanA")

assert.True(t, services[0].callbackInvokeRes, "Callback func for peer 0 should be called (chanA)")
for i := 1; i < n; i++ {
assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i)
assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i)
startsNum = 0
for i := 0; i < n; i++ {
if services[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanA")

assert.True(t, secondChannelServices[0].callbackInvokeRes, "Callback func for peer 1 should be called (chanB)")
for i := 1; i < len(secondChannelServices); i++ {
assert.False(t, secondChannelServices[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanB)", secondChannelPeerIndexes[i])
assert.False(t, secondChannelServices[i].IsLeader(), "Peer %d should not be leader in chanB", i)
startsNum = 0
for i := 0; i < len(secondChannelServices); i++ {
if secondChannelServices[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanB")

//Stopping 2 gossip instances(peer 0 and peer 1), should init re-election
//Now peer 2 become leader for first channel and peer 3 for second channel

logger.Warning("Killing 2 peers, initiation new leader election")

stopPeers(gossips[:2])

assert.True(t, waitForLeaderElection(t, secondChannelServices[1:], 0, time.Second*30, time.Second*2), "One leader (peer 2) should be selected")
assert.True(t, waitForLeaderElection(t, services[2:], 0, time.Second*30, time.Second*2), "One leader (peer 3) should be selected")
waitForFullMembership(t, gossips[2:], n-2, time.Second*30, time.Second*2)

assert.True(t, waitForLeaderElection(t, services[2:], time.Second*30, time.Second*2), "One leader should be selected after re-election - chanA")
assert.True(t, waitForLeaderElection(t, secondChannelServices[1:], time.Second*30, time.Second*2), "One leader should be selected after re-election - chanB")

assert.True(t, services[2].callbackInvokeRes, "Callback func for peer 2 should be called (chanA)")
for i := 3; i < n; i++ {
assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i)
assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i)
startsNum = 0
for i := 2; i < n; i++ {
if services[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called after re-election - chanA")

assert.True(t, secondChannelServices[1].callbackInvokeRes, "Callback func for peer 3 should be called (chanB)")
for i := 2; i < len(secondChannelServices); i++ {
assert.False(t, secondChannelServices[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanB)", secondChannelPeerIndexes[i])
assert.False(t, secondChannelServices[i].IsLeader(), "Peer %d should not be leader in chanB", i)
startsNum = 0
for i := 1; i < len(secondChannelServices); i++ {
if secondChannelServices[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called after re-election - chanB")

stopServices(secondChannelServices)
stopServices(services)
Expand Down Expand Up @@ -452,36 +480,37 @@ func waitForFullMembership(t *testing.T, gossips []GossipService, peersNum int,
return false
}

func waitForMultipleLeadersElection(t *testing.T, services []*electionService, leadersNum int, leaderIndexes []int, timeout time.Duration, testPollInterval time.Duration) bool {
func waitForMultipleLeadersElection(t *testing.T, services []*electionService, leadersNum int, timeout time.Duration, testPollInterval time.Duration) bool {
logger.Warning("Waiting for", leadersNum, "leaders")
end := time.Now().Add(timeout)
correctNumberOfLeadersFound := false
leaders := 0
for time.Now().Before(end) {
leaders := 0
incorrectLeaders := false
for i, s := range services {
leaders = 0
for _, s := range services {
if s.IsLeader() {
expectedLeader := false
for _, index := range leaderIndexes {
if i == index {
leaders++
expectedLeader = true
}
}
if !expectedLeader {
logger.Warning("Incorrect peer", i, "become leader")
incorrectLeaders = true
}
leaders++
}
}
if leaders == leadersNum && !incorrectLeaders {
return true
if leaders == leadersNum {
if correctNumberOfLeadersFound {
return true
}
correctNumberOfLeadersFound = true
} else {
correctNumberOfLeadersFound = false
}
time.Sleep(testPollInterval)
}
logger.Warning("Incorrect number of leaders", leaders)
for i, s := range services {
logger.Warning("Peer at index", i, "is leader", s.IsLeader())
}
return false
}

func waitForLeaderElection(t *testing.T, services []*electionService, leaderIndex int, timeout time.Duration, testPollInterval time.Duration) bool {
return waitForMultipleLeadersElection(t, services, 1, []int{leaderIndex}, timeout, testPollInterval)
func waitForLeaderElection(t *testing.T, services []*electionService, timeout time.Duration, testPollInterval time.Duration) bool {
return waitForMultipleLeadersElection(t, services, 1, timeout, testPollInterval)
}

func waitUntilOrFailBlocking(t *testing.T, f func(), timeout time.Duration) {
Expand Down

0 comments on commit 2724ba4

Please sign in to comment.