From e7b3c7a65f3d31cc5a19f3664c5e27b8f8beb7cd Mon Sep 17 00:00:00 2001 From: Gennady Laventman Date: Thu, 2 Mar 2017 14:03:57 +0200 Subject: [PATCH] [FAB-2576] Fix LE and gossip integration test While using real gossip as communication middleware for leader election it become impossible to point to what peer will become leader after previous leader shutdown. So test was changed to check only amount of leaders, but not what peer is a leader. Change-Id: Iac745a3e6a7e3ba66702dc4a73ca82e074d64dd0 Signed-off-by: Gennady Laventman --- gossip/service/gossip_service_test.go | 135 ++++++++++++++++---------- 1 file changed, 82 insertions(+), 53 deletions(-) diff --git a/gossip/service/gossip_service_test.go b/gossip/service/gossip_service_test.go index 32bc3f6688c..bc66f73d967 100644 --- a/gossip/service/gossip_service_test.go +++ b/gossip/service/gossip_service_test.go @@ -39,6 +39,7 @@ import ( "github.com/hyperledger/fabric/msp/mgmt/testtools" "github.com/hyperledger/fabric/peer/gossip/mcs" "github.com/hyperledger/fabric/protos/common" + "github.com/op/go-logging" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -105,7 +106,7 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) { } addPeersToChannel(t, n, 10000, channelName, gossips, peerIndexes) - waitForFullMembership(t, gossips, n, time.Second*30, time.Second*2) + waitForFullMembership(t, gossips, n, time.Second*20, time.Second*2) services := make([]*electionService, n) @@ -119,20 +120,26 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) { deliverServiceFactory.service.running[channelName] = false gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"}) + gossips[i].(*gossipServiceImpl).chains[channelName].Stop() service, exist := gossips[i].(*gossipServiceImpl).leaderElection[channelName] assert.True(t, exist, "Leader election service should be created for peer %d and channel %s", i, channelName) services[i] = &electionService{nil, false, 0} services[i].LeaderElectionService = service } - assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected") - - assert.True(t, gossips[0].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName], "Delivery service should be started in peer %d", 0) + // Is single leader was elected. + assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected") - for i := 1; i < n; i++ { - assert.False(t, gossips[i].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName], "Delivery service should not be started in peer %d", i) + startsNum := 0 + for i := 0; i < n; i++ { + // Is mockDeliverService.StartDeliverForChannel in current peer for the specific channel was invoked + if gossips[i].(*gossipServiceImpl).deliveryService.(*mockDeliverService).running[channelName] { + startsNum++ + } } + assert.Equal(t, 1, startsNum, "Only for one peer delivery client should start") + stopPeers(gossips) } @@ -170,6 +177,7 @@ func TestWithStaticDeliverClientLeader(t *testing.T) { gossips[i].(*gossipServiceImpl).deliveryFactory = deliverServiceFactory deliverServiceFactory.service.running[channelName] = false gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"}) + gossips[i].(*gossipServiceImpl).chains[channelName].Stop() } for i := 0; i < n; i++ { @@ -218,6 +226,7 @@ func TestWithStaticDeliverClientNotLeader(t *testing.T) { gossips[i].(*gossipServiceImpl).deliveryFactory = deliverServiceFactory deliverServiceFactory.service.running[channelName] = false gossips[i].InitializeChannel(channelName, &mockLedgerInfo{0}, []string{"localhost:5005"}) + gossips[i].(*gossipServiceImpl).chains[channelName].Stop() } for i := 0; i < n; i++ { @@ -302,7 +311,7 @@ func (li *mockLedgerInfo) Commit(block *common.Block) error { // Gets blocks with sequence numbers provided in the slice func (li *mockLedgerInfo) GetBlocks(blockSeqs []uint64) []*common.Block { - return nil + return make([]*common.Block, 0) } // Closes committing service @@ -310,7 +319,6 @@ func (li *mockLedgerInfo) Close() { } func TestLeaderElectionWithRealGossip(t *testing.T) { - t.Skip() // Spawn 10 gossip instances with single channel and inside same organization // Run leader election on top of each gossip instance and check that only one leader chosen @@ -319,6 +327,9 @@ func TestLeaderElectionWithRealGossip(t *testing.T) { // Check correct leader still exist for first channel and new correct leader chosen in second channel // Stop gossip instances of leader peers for both channels and see that new leader chosen for both + logging.SetLevel(logging.DEBUG, util.LoggingElectionModule) + logging.SetLevel(logging.DEBUG, util.LoggingServiceModule) + // Creating gossip service instances for peers n := 10 gossips := startPeers(t, n, 10000) @@ -335,7 +346,7 @@ func TestLeaderElectionWithRealGossip(t *testing.T) { logger.Warning("Starting leader election services") - //Stariting leader election services + //Starting leader election services services := make([]*electionService, n) for i := 0; i < n; i++ { @@ -345,13 +356,17 @@ func TestLeaderElectionWithRealGossip(t *testing.T) { logger.Warning("Waiting for leader election") - assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected") - assert.True(t, services[0].callbackInvokeRes, "Callback func for peer 0 should be called (chanA)") + assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected") - for i := 1; i < n; i++ { - assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i) - assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i) + startsNum := 0 + for i := 0; i < n; i++ { + // Is callback function was invoked by this leader election service instance + if services[i].callbackInvokeRes { + startsNum++ + } } + //Only leader should invoke callback function, so it is double check that only one leader exists + assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanA") // Adding some peers to new channel and creating leader election services for peers in new channel // Expecting peer 1 (first in list of election services) to become leader of second channel @@ -365,39 +380,52 @@ func TestLeaderElectionWithRealGossip(t *testing.T) { secondChannelServices[idx].LeaderElectionService = gossips[i].(*gossipServiceImpl).newLeaderElectionComponent(gossipCommon.ChainID(secondChannelName), secondChannelServices[idx].callback) } - assert.True(t, waitForLeaderElection(t, secondChannelServices, 0, time.Second*30, time.Second*2), "One leader (peer 1) should be selected") - assert.True(t, waitForLeaderElection(t, services, 0, time.Second*30, time.Second*2), "One leader (peer 0) should be selected") + assert.True(t, waitForLeaderElection(t, secondChannelServices, time.Second*30, time.Second*2), "One leader should be selected for chanB") + assert.True(t, waitForLeaderElection(t, services, time.Second*30, time.Second*2), "One leader should be selected for chanA") - assert.True(t, services[0].callbackInvokeRes, "Callback func for peer 0 should be called (chanA)") - for i := 1; i < n; i++ { - assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i) - assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i) + startsNum = 0 + for i := 0; i < n; i++ { + if services[i].callbackInvokeRes { + startsNum++ + } } + assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanA") - assert.True(t, secondChannelServices[0].callbackInvokeRes, "Callback func for peer 1 should be called (chanB)") - for i := 1; i < len(secondChannelServices); i++ { - assert.False(t, secondChannelServices[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanB)", secondChannelPeerIndexes[i]) - assert.False(t, secondChannelServices[i].IsLeader(), "Peer %d should not be leader in chanB", i) + startsNum = 0 + for i := 0; i < len(secondChannelServices); i++ { + if secondChannelServices[i].callbackInvokeRes { + startsNum++ + } } + assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called - chanB") //Stopping 2 gossip instances(peer 0 and peer 1), should init re-election //Now peer 2 become leader for first channel and peer 3 for second channel + + logger.Warning("Killing 2 peers, initiation new leader election") + stopPeers(gossips[:2]) - assert.True(t, waitForLeaderElection(t, secondChannelServices[1:], 0, time.Second*30, time.Second*2), "One leader (peer 2) should be selected") - assert.True(t, waitForLeaderElection(t, services[2:], 0, time.Second*30, time.Second*2), "One leader (peer 3) should be selected") + waitForFullMembership(t, gossips[2:], n-2, time.Second*30, time.Second*2) + + assert.True(t, waitForLeaderElection(t, services[2:], time.Second*30, time.Second*2), "One leader should be selected after re-election - chanA") + assert.True(t, waitForLeaderElection(t, secondChannelServices[1:], time.Second*30, time.Second*2), "One leader should be selected after re-election - chanB") - assert.True(t, services[2].callbackInvokeRes, "Callback func for peer 2 should be called (chanA)") - for i := 3; i < n; i++ { - assert.False(t, services[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanA)", i) - assert.False(t, services[i].IsLeader(), "Peer %d should not be leader in chanA", i) + startsNum = 0 + for i := 2; i < n; i++ { + if services[i].callbackInvokeRes { + startsNum++ + } } + assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called after re-election - chanA") - assert.True(t, secondChannelServices[1].callbackInvokeRes, "Callback func for peer 3 should be called (chanB)") - for i := 2; i < len(secondChannelServices); i++ { - assert.False(t, secondChannelServices[i].callbackInvokeRes, "Callback func for peer %d should not be called (chanB)", secondChannelPeerIndexes[i]) - assert.False(t, secondChannelServices[i].IsLeader(), "Peer %d should not be leader in chanB", i) + startsNum = 0 + for i := 1; i < len(secondChannelServices); i++ { + if secondChannelServices[i].callbackInvokeRes { + startsNum++ + } } + assert.Equal(t, 1, startsNum, "Only for one peer callback function should be called after re-election - chanB") stopServices(secondChannelServices) stopServices(services) @@ -452,36 +480,37 @@ func waitForFullMembership(t *testing.T, gossips []GossipService, peersNum int, return false } -func waitForMultipleLeadersElection(t *testing.T, services []*electionService, leadersNum int, leaderIndexes []int, timeout time.Duration, testPollInterval time.Duration) bool { +func waitForMultipleLeadersElection(t *testing.T, services []*electionService, leadersNum int, timeout time.Duration, testPollInterval time.Duration) bool { + logger.Warning("Waiting for", leadersNum, "leaders") end := time.Now().Add(timeout) + correctNumberOfLeadersFound := false + leaders := 0 for time.Now().Before(end) { - leaders := 0 - incorrectLeaders := false - for i, s := range services { + leaders = 0 + for _, s := range services { if s.IsLeader() { - expectedLeader := false - for _, index := range leaderIndexes { - if i == index { - leaders++ - expectedLeader = true - } - } - if !expectedLeader { - logger.Warning("Incorrect peer", i, "become leader") - incorrectLeaders = true - } + leaders++ } } - if leaders == leadersNum && !incorrectLeaders { - return true + if leaders == leadersNum { + if correctNumberOfLeadersFound { + return true + } + correctNumberOfLeadersFound = true + } else { + correctNumberOfLeadersFound = false } time.Sleep(testPollInterval) } + logger.Warning("Incorrect number of leaders", leaders) + for i, s := range services { + logger.Warning("Peer at index", i, "is leader", s.IsLeader()) + } return false } -func waitForLeaderElection(t *testing.T, services []*electionService, leaderIndex int, timeout time.Duration, testPollInterval time.Duration) bool { - return waitForMultipleLeadersElection(t, services, 1, []int{leaderIndex}, timeout, testPollInterval) +func waitForLeaderElection(t *testing.T, services []*electionService, timeout time.Duration, testPollInterval time.Duration) bool { + return waitForMultipleLeadersElection(t, services, 1, timeout, testPollInterval) } func waitUntilOrFailBlocking(t *testing.T, f func(), timeout time.Duration) {