Skip to content

Commit

Permalink
[FAB-2576] Fix LE and gossip integration test
Browse files Browse the repository at this point in the history 
While using real gossip as communication middleware for leader election
it become impossible to point to what peer will become leader after previous
leader shutdown. So test was changed to check only amount of leaders, but not
what peer is a leader.

Change-Id: Iac745a3e6a7e3ba66702dc4a73ca82e074d64dd0
Signed-off-by: Gennady Laventman <gennady@il.ibm.com>
  • Loading branch information
@gennadylaventman
gennadylaventman committed Mar 3, 2017
1 parent 94505fc commit e7b3c7a
Showing 1 changed file with 82 additions and 53 deletions.
135 changes: 82 additions & 53 deletions gossip/service/gossip_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/hyperledger/fabric/msp/mgmt/testtools"
"github.com/hyperledger/fabric/peer/gossip/mcs"
"github.com/hyperledger/fabric/protos/common"
"github.com/op/go-logging"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
Expand Down Expand Up @@ -105,7 +106,7 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) {
}
addPeersToChannel(t, n, 10000, channelName, gossips, peerIndexes)

waitForFullMembership(t, gossips, n, time.Second*30, time.Second*2)
waitForFullMembership(t, gossips, n, time.Second*20, time.Second*2)

services := make([]*electionService, n)

Expand All @@ -119,20 +120,26 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) {
deliverServiceFactory.service.running[channelName] = false

gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"})
gossips[i].(*gossipServiceImpl).chains[channelName].Stop()
service, exist := gossips[i].(*gossipServiceImpl).leaderElection[channelName]
assert.True(t, exist, "Leader election service should be created for peer %d and channel %s", i, channelName)
services[i] = &electionService{nil, false, 0}
services[i].LeaderElectionService = service
}

assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected")

assert.True(t, gossips[0].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName], "Delivery service should be started in peer %d", 0)
// Is single leader was elected.
assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected")

for i := 1; i < n; i++ {
assert.False(t, gossips[i].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName], "Delivery service should not be started in peer %d", i)
startsNum := 0
for i := 0; i < n; i++ {
// Is mockDeliverService.StartDeliverForChannel in current peer for the specific channel was invoked
if gossips[i].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName] {
startsNum++
}
}

assert.Equal(t, 1, startsNum, "Only for one peer delivery client should start")

stopPeers(gossips)
}

Expand Down Expand Up @@ -170,6 +177,7 @@ func TestWithStaticDeliverClientLeader(t *testing.T) {
gossips[i].(*gossipServiceImpl).deliveryFactory = deliverServiceFactory
deliverServiceFactory.service.running[channelName] = false
gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"})
gossips[i].(*gossipServiceImpl).chains[channelName].Stop()
}

for i := 0; i < n; i++ {
Expand Down Expand Up @@ -218,6 +226,7 @@ func TestWithStaticDeliverClientNotLeader(t *testing.T) {
gossips[i].(*gossipServiceImpl).deliveryFactory = deliverServiceFactory
deliverServiceFactory.service.running[channelName] = false
gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"})
gossips[i].(*gossipServiceImpl).chains[channelName].Stop()
}

for i := 0; i < n; i++ {
Expand Down Expand Up @@ -302,15 +311,14 @@ func (li *mockLedgerInfo) Commit(block *common.Block) error {

// Gets blocks with sequence numbers provided in the slice
func (li *mockLedgerInfo) GetBlocks(blockSeqs []uint64) []*common.Block {
return nil
return make([]*common.Block, 0)
}

// Closes committing service
func (li *mockLedgerInfo) Close() {
}

func TestLeaderElectionWithRealGossip(t *testing.T) {
t.Skip()

// Spawn 10 gossip instances with single channel and inside same organization
// Run leader election on top of each gossip instance and check that only one leader chosen
Expand All @@ -319,6 +327,9 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {
// Check correct leader still exist for first channel and new correct leader chosen in second channel
// Stop gossip instances of leader peers for both channels and see that new leader chosen for both

logging.SetLevel(logging.DEBUG, util.LoggingElectionModule)
logging.SetLevel(logging.DEBUG, util.LoggingServiceModule)

// Creating gossip service instances for peers
n := 10
gossips := startPeers(t, n, 10000)
Expand All @@ -335,7 +346,7 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {

logger.Warning("Starting leader election services")

//Stariting leader election services
//Starting leader election services
services := make([]*electionService, n)

for i := 0; i < n; i++ {
Expand All @@ -345,13 +356,17 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {

logger.Warning("Waiting for leader election")

assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected")
assert.True(t, services[0].callbackInvokeRes, "Callback func for peer 0 should be called (chanA)")
assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected")

for i := 1; i < n; i++ {
assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i)
assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i)
startsNum := 0
for i := 0; i < n; i++ {
// Is callback function was invoked by this leader election service instance
if services[i].callbackInvokeRes {
startsNum++
}
}
//Only leader should invoke callback function, so it is double check that only one leader exists
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanA")

// Adding some peers to new channel and creating leader election services for peers in new channel
// Expecting peer 1 (first in list of election services) to become leader of second channel
Expand All @@ -365,39 +380,52 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {
secondChannelServices[idx].LeaderElectionService = gossips[i].(*gossipServiceImpl).newLeaderElectionComponent(gossipCommon.ChainID(secondChannelName), secondChannelServices[idx].callback)
}

assert.True(t, waitForLeaderElection(t, secondChannelServices, 0, time.Second*30, time.Second*2), "One leader (peer 1) should be selected")
assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected")
assert.True(t, waitForLeaderElection(t, secondChannelServices, time.Second*30, time.Second*2), "One leader should be selected for chanB")
assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected for chanA")

assert.True(t, services[0].callbackInvokeRes, "Callback func for peer 0 should be called (chanA)")
for i := 1; i < n; i++ {
assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i)
assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i)
startsNum = 0
for i := 0; i < n; i++ {
if services[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanA")

assert.True(t, secondChannelServices[0].callbackInvokeRes, "Callback func for peer 1 should be called (chanB)")
for i := 1; i < len(secondChannelServices); i++ {
assert.False(t, secondChannelServices[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanB)", secondChannelPeerIndexes[i])
assert.False(t, secondChannelServices[i].IsLeader(), "Peer %d should not be leader in chanB", i)
startsNum = 0
for i := 0; i < len(secondChannelServices); i++ {
if secondChannelServices[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanB")

//Stopping 2 gossip instances(peer 0 and peer 1), should init re-election
//Now peer 2 become leader for first channel and peer 3 for second channel

logger.Warning("Killing 2 peers, initiation new leader election")

stopPeers(gossips[:2])

assert.True(t, waitForLeaderElection(t, secondChannelServices[1:], 0, time.Second*30, time.Second*2), "One leader (peer 2) should be selected")
assert.True(t, waitForLeaderElection(t, services[2:], 0, time.Second*30, time.Second*2), "One leader (peer 3) should be selected")
waitForFullMembership(t, gossips[2:], n-2, time.Second*30, time.Second*2)

assert.True(t, waitForLeaderElection(t, services[2:], time.Second*30, time.Second*2), "One leader should be selected after re-election - chanA")
assert.True(t, waitForLeaderElection(t, secondChannelServices[1:], time.Second*30, time.Second*2), "One leader should be selected after re-election - chanB")

assert.True(t, services[2].callbackInvokeRes, "Callback func for peer 2 should be called (chanA)")
for i := 3; i < n; i++ {
assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i)
assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i)
startsNum = 0
for i := 2; i < n; i++ {
if services[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called after re-election - chanA")

assert.True(t, secondChannelServices[1].callbackInvokeRes, "Callback func for peer 3 should be called (chanB)")
for i := 2; i < len(secondChannelServices); i++ {
assert.False(t, secondChannelServices[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanB)", secondChannelPeerIndexes[i])
assert.False(t, secondChannelServices[i].IsLeader(), "Peer %d should not be leader in chanB", i)
startsNum = 0
for i := 1; i < len(secondChannelServices); i++ {
if secondChannelServices[i].callbackInvokeRes {
startsNum++
}
}
assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called after re-election - chanB")

stopServices(secondChannelServices)
stopServices(services)
Expand Down Expand Up @@ -452,36 +480,37 @@ func waitForFullMembership(t *testing.T, gossips []GossipService, peersNum int,
return false
}

func waitForMultipleLeadersElection(t *testing.T, services []*electionService, leadersNum int, leaderIndexes []int, timeout time.Duration, testPollInterval time.Duration) bool {
func waitForMultipleLeadersElection(t *testing.T, services []*electionService, leadersNum int, timeout time.Duration, testPollInterval time.Duration) bool {
logger.Warning("Waiting for", leadersNum, "leaders")
end := time.Now().Add(timeout)
correctNumberOfLeadersFound := false
leaders := 0
for time.Now().Before(end) {
leaders := 0
incorrectLeaders := false
for i, s := range services {
leaders = 0
for _, s := range services {
if s.IsLeader() {
expectedLeader := false
for _, index := range leaderIndexes {
if i == index {
leaders++
expectedLeader = true
}
}
if !expectedLeader {
logger.Warning("Incorrect peer", i, "become leader")
incorrectLeaders = true
}
leaders++
}
}
if leaders == leadersNum && !incorrectLeaders {
return true
if leaders == leadersNum {
if correctNumberOfLeadersFound {
return true
}
correctNumberOfLeadersFound = true
} else {
correctNumberOfLeadersFound = false
}
time.Sleep(testPollInterval)
}
logger.Warning("Incorrect number of leaders", leaders)
for i, s := range services {
logger.Warning("Peer at index", i, "is leader", s.IsLeader())
}
return false
}

func waitForLeaderElection(t *testing.T, services []*electionService, leaderIndex int, timeout time.Duration, testPollInterval time.Duration) bool {
return waitForMultipleLeadersElection(t, services, 1, []int{leaderIndex}, timeout, testPollInterval)
func waitForLeaderElection(t *testing.T, services []*electionService, timeout time.Duration, testPollInterval time.Duration) bool {
return waitForMultipleLeadersElection(t, services, 1, timeout, testPollInterval)
}

func waitUntilOrFailBlocking(t *testing.T, f func(), timeout time.Duration) {
Expand Down

0 comments on commit e7b3c7a

Please sign in to comment.