From 6432c503653182b58fea85f6f923dbb52f991e2f Mon Sep 17 00:00:00 2001 From: Will Lahti Date: Mon, 19 Oct 2020 16:31:20 -0400 Subject: [PATCH] Chan.Part.API: channel participation relation/status metrics FAB-18087 Signed-off-by: Will Lahti --- common/metrics/provider.go | 2 +- docs/source/metrics_reference.rst | 14 ++ .../mocks/channel_management.go | 217 +++++++++------- orderer/common/follower/follower_chain.go | 53 ++-- .../common/follower/follower_chain_test.go | 74 +++--- .../channel_participation_metrics_reporter.go | 80 ++++++ orderer/common/multichannel/chainsupport.go | 3 + orderer/common/multichannel/metrics.go | 92 +++++++ orderer/common/multichannel/metrics_test.go | 167 +++++++++++++ .../multichannel/mocks/metrics_provider.go | 61 +++++ .../multichannel/multichannel_suite_test.go | 19 ++ orderer/common/multichannel/registrar.go | 54 ++-- orderer/common/multichannel/registrar_test.go | 27 ++ orderer/consensus/etcdraft/consenter.go | 9 +- orderer/consensus/etcdraft/consenter_test.go | 20 +- .../consensus/etcdraft/mocks/chain_manager.go | 236 ++++++++++++++++-- 16 files changed, 946 insertions(+), 182 deletions(-) create mode 100644 orderer/common/follower/mocks/channel_participation_metrics_reporter.go create mode 100644 orderer/common/multichannel/metrics.go create mode 100644 orderer/common/multichannel/metrics_test.go create mode 100644 orderer/common/multichannel/mocks/metrics_provider.go create mode 100644 orderer/common/multichannel/multichannel_suite_test.go diff --git a/common/metrics/provider.go b/common/metrics/provider.go index 537bcd0c61a..21cfe46a1f4 100644 --- a/common/metrics/provider.go +++ b/common/metrics/provider.go @@ -74,7 +74,7 @@ type Gauge interface { // Add increments a Gauge value. Add(delta float64) // TODO: consider removing - // Set is used to update the current value associted with a Gauge. + // Set is used to update the current value associated with a Gauge. Set(value float64) } diff --git a/docs/source/metrics_reference.rst b/docs/source/metrics_reference.rst index 117e5e7f7bf..f5a76264435 100644 --- a/docs/source/metrics_reference.rst +++ b/docs/source/metrics_reference.rst @@ -193,6 +193,13 @@ The following orderer metrics are exported for consumption by Prometheus. +----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+ | logging_entries_written | counter | Number of log entries that are written | level | | +----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+ +| participation_cluster_relation | gauge | The channel participation cluster relation of the node: 0 | channel | | +| | | if none, 1 if consenter, 2 if follower, 3 if | | | +| | | config-tracker. | | | ++----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+ +| participation_status | gauge | The channel participation status of the node: 0 if | channel | | +| | | inactive, 1 if active, 2 if onboarding. | | | ++----------------------------------------------+-----------+------------------------------------------------------------+-----------+--------------------------------------------------------------------+ StatsD ~~~~~~ @@ -326,6 +333,13 @@ associated with the metric. +---------------------------------------------------------------------------+-----------+------------------------------------------------------------+ | logging.entries_written.%{level} | counter | Number of log entries that are written | +---------------------------------------------------------------------------+-----------+------------------------------------------------------------+ +| participation.cluster_relation.%{channel} | gauge | The channel participation cluster relation of the node: 0 | +| | | if none, 1 if consenter, 2 if follower, 3 if | +| | | config-tracker. | ++---------------------------------------------------------------------------+-----------+------------------------------------------------------------+ +| participation.status.%{channel} | gauge | The channel participation status of the node: 0 if | +| | | inactive, 1 if active, 2 if onboarding. | ++---------------------------------------------------------------------------+-----------+------------------------------------------------------------+ Peer Metrics ------------ diff --git a/orderer/common/channelparticipation/mocks/channel_management.go b/orderer/common/channelparticipation/mocks/channel_management.go index 4088cacf5b3..e8becf191d8 100644 --- a/orderer/common/channelparticipation/mocks/channel_management.go +++ b/orderer/common/channelparticipation/mocks/channel_management.go @@ -4,25 +4,16 @@ package mocks import ( "sync" - cb "github.com/hyperledger/fabric-protos-go/common" + "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric/orderer/common/channelparticipation" "github.com/hyperledger/fabric/orderer/common/types" ) type ChannelManagement struct { - ChannelListStub func() types.ChannelList - channelListMutex sync.RWMutex - channelListArgsForCall []struct{} - channelListReturns struct { - result1 types.ChannelList - } - channelListReturnsOnCall map[int]struct { - result1 types.ChannelList - } - ChannelInfoStub func(channelID string) (types.ChannelInfo, error) + ChannelInfoStub func(string) (types.ChannelInfo, error) channelInfoMutex sync.RWMutex channelInfoArgsForCall []struct { - channelID string + arg1 string } channelInfoReturns struct { result1 types.ChannelInfo @@ -32,12 +23,22 @@ type ChannelManagement struct { result1 types.ChannelInfo result2 error } - JoinChannelStub func(channelID string, configBlock *cb.Block, isAppChannel bool) (types.ChannelInfo, error) + ChannelListStub func() types.ChannelList + channelListMutex sync.RWMutex + channelListArgsForCall []struct { + } + channelListReturns struct { + result1 types.ChannelList + } + channelListReturnsOnCall map[int]struct { + result1 types.ChannelList + } + JoinChannelStub func(string, *common.Block, bool) (types.ChannelInfo, error) joinChannelMutex sync.RWMutex joinChannelArgsForCall []struct { - channelID string - configBlock *cb.Block - isAppChannel bool + arg1 string + arg2 *common.Block + arg3 bool } joinChannelReturns struct { result1 types.ChannelInfo @@ -47,10 +48,10 @@ type ChannelManagement struct { result1 types.ChannelInfo result2 error } - RemoveChannelStub func(channelID string) error + RemoveChannelStub func(string) error removeChannelMutex sync.RWMutex removeChannelArgsForCall []struct { - channelID string + arg1 string } removeChannelReturns struct { result1 error @@ -62,61 +63,22 @@ type ChannelManagement struct { invocationsMutex sync.RWMutex } -func (fake *ChannelManagement) ChannelList() types.ChannelList { - fake.channelListMutex.Lock() - ret, specificReturn := fake.channelListReturnsOnCall[len(fake.channelListArgsForCall)] - fake.channelListArgsForCall = append(fake.channelListArgsForCall, struct{}{}) - fake.recordInvocation("ChannelList", []interface{}{}) - fake.channelListMutex.Unlock() - if fake.ChannelListStub != nil { - return fake.ChannelListStub() - } - if specificReturn { - return ret.result1 - } - return fake.channelListReturns.result1 -} - -func (fake *ChannelManagement) ChannelListCallCount() int { - fake.channelListMutex.RLock() - defer fake.channelListMutex.RUnlock() - return len(fake.channelListArgsForCall) -} - -func (fake *ChannelManagement) ChannelListReturns(result1 types.ChannelList) { - fake.ChannelListStub = nil - fake.channelListReturns = struct { - result1 types.ChannelList - }{result1} -} - -func (fake *ChannelManagement) ChannelListReturnsOnCall(i int, result1 types.ChannelList) { - fake.ChannelListStub = nil - if fake.channelListReturnsOnCall == nil { - fake.channelListReturnsOnCall = make(map[int]struct { - result1 types.ChannelList - }) - } - fake.channelListReturnsOnCall[i] = struct { - result1 types.ChannelList - }{result1} -} - -func (fake *ChannelManagement) ChannelInfo(channelID string) (types.ChannelInfo, error) { +func (fake *ChannelManagement) ChannelInfo(arg1 string) (types.ChannelInfo, error) { fake.channelInfoMutex.Lock() ret, specificReturn := fake.channelInfoReturnsOnCall[len(fake.channelInfoArgsForCall)] fake.channelInfoArgsForCall = append(fake.channelInfoArgsForCall, struct { - channelID string - }{channelID}) - fake.recordInvocation("ChannelInfo", []interface{}{channelID}) + arg1 string + }{arg1}) + fake.recordInvocation("ChannelInfo", []interface{}{arg1}) fake.channelInfoMutex.Unlock() if fake.ChannelInfoStub != nil { - return fake.ChannelInfoStub(channelID) + return fake.ChannelInfoStub(arg1) } if specificReturn { return ret.result1, ret.result2 } - return fake.channelInfoReturns.result1, fake.channelInfoReturns.result2 + fakeReturns := fake.channelInfoReturns + return fakeReturns.result1, fakeReturns.result2 } func (fake *ChannelManagement) ChannelInfoCallCount() int { @@ -125,13 +87,22 @@ func (fake *ChannelManagement) ChannelInfoCallCount() int { return len(fake.channelInfoArgsForCall) } +func (fake *ChannelManagement) ChannelInfoCalls(stub func(string) (types.ChannelInfo, error)) { + fake.channelInfoMutex.Lock() + defer fake.channelInfoMutex.Unlock() + fake.ChannelInfoStub = stub +} + func (fake *ChannelManagement) ChannelInfoArgsForCall(i int) string { fake.channelInfoMutex.RLock() defer fake.channelInfoMutex.RUnlock() - return fake.channelInfoArgsForCall[i].channelID + argsForCall := fake.channelInfoArgsForCall[i] + return argsForCall.arg1 } func (fake *ChannelManagement) ChannelInfoReturns(result1 types.ChannelInfo, result2 error) { + fake.channelInfoMutex.Lock() + defer fake.channelInfoMutex.Unlock() fake.ChannelInfoStub = nil fake.channelInfoReturns = struct { result1 types.ChannelInfo @@ -140,6 +111,8 @@ func (fake *ChannelManagement) ChannelInfoReturns(result1 types.ChannelInfo, res } func (fake *ChannelManagement) ChannelInfoReturnsOnCall(i int, result1 types.ChannelInfo, result2 error) { + fake.channelInfoMutex.Lock() + defer fake.channelInfoMutex.Unlock() fake.ChannelInfoStub = nil if fake.channelInfoReturnsOnCall == nil { fake.channelInfoReturnsOnCall = make(map[int]struct { @@ -153,23 +126,76 @@ func (fake *ChannelManagement) ChannelInfoReturnsOnCall(i int, result1 types.Cha }{result1, result2} } -func (fake *ChannelManagement) JoinChannel(channelID string, configBlock *cb.Block, isAppChannel bool) (types.ChannelInfo, error) { +func (fake *ChannelManagement) ChannelList() types.ChannelList { + fake.channelListMutex.Lock() + ret, specificReturn := fake.channelListReturnsOnCall[len(fake.channelListArgsForCall)] + fake.channelListArgsForCall = append(fake.channelListArgsForCall, struct { + }{}) + fake.recordInvocation("ChannelList", []interface{}{}) + fake.channelListMutex.Unlock() + if fake.ChannelListStub != nil { + return fake.ChannelListStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.channelListReturns + return fakeReturns.result1 +} + +func (fake *ChannelManagement) ChannelListCallCount() int { + fake.channelListMutex.RLock() + defer fake.channelListMutex.RUnlock() + return len(fake.channelListArgsForCall) +} + +func (fake *ChannelManagement) ChannelListCalls(stub func() types.ChannelList) { + fake.channelListMutex.Lock() + defer fake.channelListMutex.Unlock() + fake.ChannelListStub = stub +} + +func (fake *ChannelManagement) ChannelListReturns(result1 types.ChannelList) { + fake.channelListMutex.Lock() + defer fake.channelListMutex.Unlock() + fake.ChannelListStub = nil + fake.channelListReturns = struct { + result1 types.ChannelList + }{result1} +} + +func (fake *ChannelManagement) ChannelListReturnsOnCall(i int, result1 types.ChannelList) { + fake.channelListMutex.Lock() + defer fake.channelListMutex.Unlock() + fake.ChannelListStub = nil + if fake.channelListReturnsOnCall == nil { + fake.channelListReturnsOnCall = make(map[int]struct { + result1 types.ChannelList + }) + } + fake.channelListReturnsOnCall[i] = struct { + result1 types.ChannelList + }{result1} +} + +func (fake *ChannelManagement) JoinChannel(arg1 string, arg2 *common.Block, arg3 bool) (types.ChannelInfo, error) { fake.joinChannelMutex.Lock() ret, specificReturn := fake.joinChannelReturnsOnCall[len(fake.joinChannelArgsForCall)] fake.joinChannelArgsForCall = append(fake.joinChannelArgsForCall, struct { - channelID string - configBlock *cb.Block - isAppChannel bool - }{channelID, configBlock, isAppChannel}) - fake.recordInvocation("JoinChannel", []interface{}{channelID, configBlock, isAppChannel}) + arg1 string + arg2 *common.Block + arg3 bool + }{arg1, arg2, arg3}) + fake.recordInvocation("JoinChannel", []interface{}{arg1, arg2, arg3}) fake.joinChannelMutex.Unlock() if fake.JoinChannelStub != nil { - return fake.JoinChannelStub(channelID, configBlock, isAppChannel) + return fake.JoinChannelStub(arg1, arg2, arg3) } if specificReturn { return ret.result1, ret.result2 } - return fake.joinChannelReturns.result1, fake.joinChannelReturns.result2 + fakeReturns := fake.joinChannelReturns + return fakeReturns.result1, fakeReturns.result2 } func (fake *ChannelManagement) JoinChannelCallCount() int { @@ -178,13 +204,22 @@ func (fake *ChannelManagement) JoinChannelCallCount() int { return len(fake.joinChannelArgsForCall) } -func (fake *ChannelManagement) JoinChannelArgsForCall(i int) (string, *cb.Block, bool) { +func (fake *ChannelManagement) JoinChannelCalls(stub func(string, *common.Block, bool) (types.ChannelInfo, error)) { + fake.joinChannelMutex.Lock() + defer fake.joinChannelMutex.Unlock() + fake.JoinChannelStub = stub +} + +func (fake *ChannelManagement) JoinChannelArgsForCall(i int) (string, *common.Block, bool) { fake.joinChannelMutex.RLock() defer fake.joinChannelMutex.RUnlock() - return fake.joinChannelArgsForCall[i].channelID, fake.joinChannelArgsForCall[i].configBlock, fake.joinChannelArgsForCall[i].isAppChannel + argsForCall := fake.joinChannelArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *ChannelManagement) JoinChannelReturns(result1 types.ChannelInfo, result2 error) { + fake.joinChannelMutex.Lock() + defer fake.joinChannelMutex.Unlock() fake.JoinChannelStub = nil fake.joinChannelReturns = struct { result1 types.ChannelInfo @@ -193,6 +228,8 @@ func (fake *ChannelManagement) JoinChannelReturns(result1 types.ChannelInfo, res } func (fake *ChannelManagement) JoinChannelReturnsOnCall(i int, result1 types.ChannelInfo, result2 error) { + fake.joinChannelMutex.Lock() + defer fake.joinChannelMutex.Unlock() fake.JoinChannelStub = nil if fake.joinChannelReturnsOnCall == nil { fake.joinChannelReturnsOnCall = make(map[int]struct { @@ -206,21 +243,22 @@ func (fake *ChannelManagement) JoinChannelReturnsOnCall(i int, result1 types.Cha }{result1, result2} } -func (fake *ChannelManagement) RemoveChannel(channelID string) error { +func (fake *ChannelManagement) RemoveChannel(arg1 string) error { fake.removeChannelMutex.Lock() ret, specificReturn := fake.removeChannelReturnsOnCall[len(fake.removeChannelArgsForCall)] fake.removeChannelArgsForCall = append(fake.removeChannelArgsForCall, struct { - channelID string - }{channelID}) - fake.recordInvocation("RemoveChannel", []interface{}{channelID}) + arg1 string + }{arg1}) + fake.recordInvocation("RemoveChannel", []interface{}{arg1}) fake.removeChannelMutex.Unlock() if fake.RemoveChannelStub != nil { - return fake.RemoveChannelStub(channelID) + return fake.RemoveChannelStub(arg1) } if specificReturn { return ret.result1 } - return fake.removeChannelReturns.result1 + fakeReturns := fake.removeChannelReturns + return fakeReturns.result1 } func (fake *ChannelManagement) RemoveChannelCallCount() int { @@ -229,13 +267,22 @@ func (fake *ChannelManagement) RemoveChannelCallCount() int { return len(fake.removeChannelArgsForCall) } +func (fake *ChannelManagement) RemoveChannelCalls(stub func(string) error) { + fake.removeChannelMutex.Lock() + defer fake.removeChannelMutex.Unlock() + fake.RemoveChannelStub = stub +} + func (fake *ChannelManagement) RemoveChannelArgsForCall(i int) string { fake.removeChannelMutex.RLock() defer fake.removeChannelMutex.RUnlock() - return fake.removeChannelArgsForCall[i].channelID + argsForCall := fake.removeChannelArgsForCall[i] + return argsForCall.arg1 } func (fake *ChannelManagement) RemoveChannelReturns(result1 error) { + fake.removeChannelMutex.Lock() + defer fake.removeChannelMutex.Unlock() fake.RemoveChannelStub = nil fake.removeChannelReturns = struct { result1 error @@ -243,6 +290,8 @@ func (fake *ChannelManagement) RemoveChannelReturns(result1 error) { } func (fake *ChannelManagement) RemoveChannelReturnsOnCall(i int, result1 error) { + fake.removeChannelMutex.Lock() + defer fake.removeChannelMutex.Unlock() fake.RemoveChannelStub = nil if fake.removeChannelReturnsOnCall == nil { fake.removeChannelReturnsOnCall = make(map[int]struct { @@ -257,10 +306,10 @@ func (fake *ChannelManagement) RemoveChannelReturnsOnCall(i int, result1 error) func (fake *ChannelManagement) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.channelListMutex.RLock() - defer fake.channelListMutex.RUnlock() fake.channelInfoMutex.RLock() defer fake.channelInfoMutex.RUnlock() + fake.channelListMutex.RLock() + defer fake.channelListMutex.RUnlock() fake.joinChannelMutex.RLock() defer fake.joinChannelMutex.RUnlock() fake.removeChannelMutex.RLock() diff --git a/orderer/common/follower/follower_chain.go b/orderer/common/follower/follower_chain.go index 0da87d5594e..6285bb57fd2 100644 --- a/orderer/common/follower/follower_chain.go +++ b/orderer/common/follower/follower_chain.go @@ -62,6 +62,12 @@ type ChainCreator interface { SwitchFollowerToChain(chainName string) } +//go:generate counterfeiter -o mocks/channel_participation_metrics_reporter.go -fake-name ChannelParticipationMetricsReporter . ChannelParticipationMetricsReporter + +type ChannelParticipationMetricsReporter interface { + ReportRelationAndStatusMetrics(channelID string, relation types.ClusterRelation, status types.Status) +} + // Chain implements a component that allows the orderer to follow a specific channel when is not a cluster member, // that is, be a "follower" of the cluster. It also allows the orderer to perform "onboarding" for // channels it is joining as a member, with a join-block. @@ -115,6 +121,8 @@ type Chain struct { chainCreator ChainCreator cryptoProvider bccsp.BCCSP // Cryptographic services + + channelParticipationMetricsReporter ChannelParticipationMetricsReporter } // NewChain constructs a follower.Chain object. @@ -126,24 +134,26 @@ func NewChain( blockPullerFactory BlockPullerFactory, chainCreator ChainCreator, cryptoProvider bccsp.BCCSP, + channelParticipationMetricsReporter ChannelParticipationMetricsReporter, ) (*Chain, error) { options.applyDefaults() chain := &Chain{ - stopChan: make(chan struct{}), - doneChan: make(chan struct{}), - cRel: types.ClusterRelationFollower, - status: types.StatusOnBoarding, - ledgerResources: ledgerResources, - clusterConsenter: clusterConsenter, - joinBlock: joinBlock, - firstHeight: ledgerResources.Height(), - options: options, - logger: options.Logger.With("channel", ledgerResources.ChannelID()), - timeAfter: options.TimeAfter, - blockPullerFactory: blockPullerFactory, - chainCreator: chainCreator, - cryptoProvider: cryptoProvider, + stopChan: make(chan struct{}), + doneChan: make(chan struct{}), + cRel: types.ClusterRelationFollower, + status: types.StatusOnBoarding, + ledgerResources: ledgerResources, + clusterConsenter: clusterConsenter, + joinBlock: joinBlock, + firstHeight: ledgerResources.Height(), + options: options, + logger: options.Logger.With("channel", ledgerResources.ChannelID()), + timeAfter: options.TimeAfter, + blockPullerFactory: blockPullerFactory, + chainCreator: chainCreator, + cryptoProvider: cryptoProvider, + channelParticipationMetricsReporter: channelParticipationMetricsReporter, } if ledgerResources.Height() > 0 { @@ -190,6 +200,8 @@ func NewChain( chain.logger.Debugf("Options are: %v", chain.options) + chain.channelParticipationMetricsReporter.ReportRelationAndStatusMetrics(ledgerResources.ChannelID(), chain.cRel, chain.status) + return chain, nil } @@ -244,6 +256,8 @@ func (c *Chain) setStatus(status types.Status) { defer c.mutex.Unlock() c.status = status + + c.channelParticipationMetricsReporter.ReportRelationAndStatusMetrics(c.ledgerResources.ChannelID(), c.cRel, c.status) } func (c *Chain) setClusterRelation(clusterRelation types.ClusterRelation) { @@ -251,6 +265,8 @@ func (c *Chain) setClusterRelation(clusterRelation types.ClusterRelation) { defer c.mutex.Unlock() c.cRel = clusterRelation + + c.channelParticipationMetricsReporter.ReportRelationAndStatusMetrics(c.ledgerResources.ChannelID(), c.cRel, c.status) } func (c *Chain) Height() uint64 { @@ -374,10 +390,11 @@ func (c *Chain) pullUpToJoin() error { return nil } -// pullAfterJoin pulls blocks continuously, inspecting the fetched config blocks for membership. -// On every config block, it renews the BlockPuller, to take in the new configuration. -// It will exit with 'nil' if it detects a config block that indicates the orderer is a member of the cluster. -// It checks whether the chain was stopped between blocks. +// pullAfterJoin pulls blocks continuously, inspecting the fetched config +// blocks for membership. On every config block, it renews the BlockPuller, +// to take in the new configuration. It will exit with 'nil' if it detects +// a config block that indicates the orderer is a member of the cluster. It +// checks whether the chain was stopped between blocks. func (c *Chain) pullAfterJoin() error { c.setStatus(types.StatusActive) diff --git a/orderer/common/follower/follower_chain_test.go b/orderer/common/follower/follower_chain_test.go index 4eb484af804..fe6e00c5895 100644 --- a/orderer/common/follower/follower_chain_test.go +++ b/orderer/common/follower/follower_chain_test.go @@ -43,17 +43,18 @@ var testLogger = flogging.MustGetLogger("follower.test") // Global test variables var ( - cryptoProvider bccsp.BCCSP - localBlockchain *memoryBlockChain - remoteBlockchain *memoryBlockChain - ledgerResources *mocks.LedgerResources - mockClusterConsenter *mocks.ClusterConsenter - pullerFactory *mocks.BlockPullerFactory - puller *mocks.ChannelPuller - mockChainCreator *mocks.ChainCreator - options follower.Options - timeAfterCount *mocks.TimeAfter - maxDelay int64 + cryptoProvider bccsp.BCCSP + localBlockchain *memoryBlockChain + remoteBlockchain *memoryBlockChain + ledgerResources *mocks.LedgerResources + mockClusterConsenter *mocks.ClusterConsenter + mockChannelParticipationMetricsReporter *mocks.ChannelParticipationMetricsReporter + pullerFactory *mocks.BlockPullerFactory + puller *mocks.ChannelPuller + mockChainCreator *mocks.ChainCreator + options follower.Options + timeAfterCount *mocks.TimeAfter + maxDelay int64 ) // Before each test in all test cases @@ -78,6 +79,8 @@ func globalSetup(t *testing.T) { mockChainCreator = &mocks.ChainCreator{} + mockChannelParticipationMetricsReporter = &mocks.ChannelParticipationMetricsReporter{} + options = follower.Options{ Logger: testLogger, PullRetryMinInterval: 1 * time.Microsecond, @@ -109,7 +112,7 @@ func TestFollowerNewChain(t *testing.T) { globalSetup(t) ledgerResources.HeightReturns(0) mockClusterConsenter.IsChannelMemberReturns(false, nil) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, nil) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, nil, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -121,7 +124,7 @@ func TestFollowerNewChain(t *testing.T) { globalSetup(t) ledgerResources.HeightReturns(0) mockClusterConsenter.IsChannelMemberReturns(true, nil) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, nil) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, nil, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -137,10 +140,10 @@ func TestFollowerNewChain(t *testing.T) { t.Run("bad join block", func(t *testing.T) { globalSetup(t) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, &common.Block{}, options, pullerFactory, mockChainCreator, nil) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, &common.Block{}, options, pullerFactory, mockChainCreator, nil, mockChannelParticipationMetricsReporter) require.EqualError(t, err, "block header is nil") require.Nil(t, chain) - chain, err = follower.NewChain(ledgerResources, mockClusterConsenter, &common.Block{Header: &common.BlockHeader{}}, options, pullerFactory, mockChainCreator, nil) + chain, err = follower.NewChain(ledgerResources, mockClusterConsenter, &common.Block{Header: &common.BlockHeader{}}, options, pullerFactory, mockChainCreator, nil, mockChannelParticipationMetricsReporter) require.EqualError(t, err, "block data is nil") require.Nil(t, chain) }) @@ -149,7 +152,7 @@ func TestFollowerNewChain(t *testing.T) { globalSetup(t) localBlockchain.fill(5) mockClusterConsenter.IsChannelMemberCalls(amIReallyInChannel) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, nil) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, nil, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -187,7 +190,7 @@ func TestFollowerPullUpToJoin(t *testing.T) { setup() mockClusterConsenter.IsChannelMemberCalls(amIReallyInChannel) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -217,13 +220,19 @@ func TestFollowerPullUpToJoin(t *testing.T) { require.True(t, joinBlockAppRaft.Header.Number > ledgerResources.Height()) require.True(t, ledgerResources.Height() > 0) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() require.Equal(t, types.ClusterRelationConsenter, cRel) require.Equal(t, types.StatusOnBoarding, status) + require.Equal(t, 1, mockChannelParticipationMetricsReporter.ReportRelationAndStatusMetricsCallCount()) + channel, relation, status := mockChannelParticipationMetricsReporter.ReportRelationAndStatusMetricsArgsForCall(0) + require.Equal(t, "my-channel", channel) + require.Equal(t, types.ClusterRelationConsenter, relation) + require.Equal(t, types.StatusOnBoarding, status) + require.NotPanics(t, chain.Start) wgChain.Wait() require.NotPanics(t, chain.Halt) @@ -239,6 +248,13 @@ func TestFollowerPullUpToJoin(t *testing.T) { require.Equal(t, remoteBlockchain.Block(i).Header, localBlockchain.Block(i).Header, "failed block i=%d", i) } require.Equal(t, 1, mockChainCreator.SwitchFollowerToChainCallCount()) + + require.Equal(t, 3, mockChannelParticipationMetricsReporter.ReportRelationAndStatusMetricsCallCount()) + channel, relation, status = mockChannelParticipationMetricsReporter.ReportRelationAndStatusMetricsArgsForCall(2) + require.Equal(t, "my-channel", channel) + require.Equal(t, types.ClusterRelationConsenter, relation) + require.Equal(t, types.StatusActive, status) + }) t.Run("no need to pull, member", func(t *testing.T) { setup() @@ -247,7 +263,7 @@ func TestFollowerPullUpToJoin(t *testing.T) { localBlockchain.appendConfig(1) //No gap between the ledger and the join block require.True(t, joinBlockAppRaft.Header.Number < ledgerResources.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -286,7 +302,7 @@ func TestFollowerPullUpToJoin(t *testing.T) { pullerFactory.BlockPullerReturns(puller, nil) options.TimeAfter = timeAfterCount.After - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -351,7 +367,7 @@ func TestFollowerPullAfterJoin(t *testing.T) { }) require.Equal(t, joinNum+1, localBlockchain.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -391,7 +407,7 @@ func TestFollowerPullAfterJoin(t *testing.T) { }) require.Equal(t, joinNum+1, localBlockchain.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) @@ -447,7 +463,7 @@ func TestFollowerPullAfterJoin(t *testing.T) { }) require.Equal(t, joinNum+1, localBlockchain.Height()) options.TimeAfter = timeAfterCount.After - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) @@ -497,7 +513,7 @@ func TestFollowerPullAfterJoin(t *testing.T) { mockChainCreator.SwitchFollowerToChainCalls(func(_ string) { wgChain.Done() }) //Stop when a new chain is created require.Equal(t, joinNum+1, localBlockchain.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -570,7 +586,7 @@ func TestFollowerPullAfterJoin(t *testing.T) { options.TimeAfter = timeAfterCount.After - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, nil, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) @@ -642,7 +658,7 @@ func TestFollowerPullPastJoin(t *testing.T) { }) require.Equal(t, uint64(0), localBlockchain.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -682,7 +698,7 @@ func TestFollowerPullPastJoin(t *testing.T) { }) require.Equal(t, uint64(0), localBlockchain.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) @@ -730,7 +746,7 @@ func TestFollowerPullPastJoin(t *testing.T) { mockChainCreator.SwitchFollowerToChainCalls(func(_ string) { wgChain.Done() }) //Stop when a new chain is created require.Equal(t, uint64(6), localBlockchain.Height()) - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) cRel, status := chain.StatusReport() @@ -803,7 +819,7 @@ func TestFollowerPullPastJoin(t *testing.T) { options.TimeAfter = timeAfterCount.After - chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider) + chain, err := follower.NewChain(ledgerResources, mockClusterConsenter, joinBlockAppRaft, options, pullerFactory, mockChainCreator, cryptoProvider, mockChannelParticipationMetricsReporter) require.NoError(t, err) diff --git a/orderer/common/follower/mocks/channel_participation_metrics_reporter.go b/orderer/common/follower/mocks/channel_participation_metrics_reporter.go new file mode 100644 index 00000000000..744852c2391 --- /dev/null +++ b/orderer/common/follower/mocks/channel_participation_metrics_reporter.go @@ -0,0 +1,80 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "sync" + + "github.com/hyperledger/fabric/orderer/common/follower" + "github.com/hyperledger/fabric/orderer/common/types" +) + +type ChannelParticipationMetricsReporter struct { + ReportRelationAndStatusMetricsStub func(string, types.ClusterRelation, types.Status) + reportRelationAndStatusMetricsMutex sync.RWMutex + reportRelationAndStatusMetricsArgsForCall []struct { + arg1 string + arg2 types.ClusterRelation + arg3 types.Status + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *ChannelParticipationMetricsReporter) ReportRelationAndStatusMetrics(arg1 string, arg2 types.ClusterRelation, arg3 types.Status) { + fake.reportRelationAndStatusMetricsMutex.Lock() + fake.reportRelationAndStatusMetricsArgsForCall = append(fake.reportRelationAndStatusMetricsArgsForCall, struct { + arg1 string + arg2 types.ClusterRelation + arg3 types.Status + }{arg1, arg2, arg3}) + fake.recordInvocation("ReportRelationAndStatusMetrics", []interface{}{arg1, arg2, arg3}) + fake.reportRelationAndStatusMetricsMutex.Unlock() + if fake.ReportRelationAndStatusMetricsStub != nil { + fake.ReportRelationAndStatusMetricsStub(arg1, arg2, arg3) + } +} + +func (fake *ChannelParticipationMetricsReporter) ReportRelationAndStatusMetricsCallCount() int { + fake.reportRelationAndStatusMetricsMutex.RLock() + defer fake.reportRelationAndStatusMetricsMutex.RUnlock() + return len(fake.reportRelationAndStatusMetricsArgsForCall) +} + +func (fake *ChannelParticipationMetricsReporter) ReportRelationAndStatusMetricsCalls(stub func(string, types.ClusterRelation, types.Status)) { + fake.reportRelationAndStatusMetricsMutex.Lock() + defer fake.reportRelationAndStatusMetricsMutex.Unlock() + fake.ReportRelationAndStatusMetricsStub = stub +} + +func (fake *ChannelParticipationMetricsReporter) ReportRelationAndStatusMetricsArgsForCall(i int) (string, types.ClusterRelation, types.Status) { + fake.reportRelationAndStatusMetricsMutex.RLock() + defer fake.reportRelationAndStatusMetricsMutex.RUnlock() + argsForCall := fake.reportRelationAndStatusMetricsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *ChannelParticipationMetricsReporter) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.reportRelationAndStatusMetricsMutex.RLock() + defer fake.reportRelationAndStatusMetricsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *ChannelParticipationMetricsReporter) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ follower.ChannelParticipationMetricsReporter = new(ChannelParticipationMetricsReporter) diff --git a/orderer/common/multichannel/chainsupport.go b/orderer/common/multichannel/chainsupport.go index 46fde4c16c1..b0f68acf9f1 100644 --- a/orderer/common/multichannel/chainsupport.go +++ b/orderer/common/multichannel/chainsupport.go @@ -99,6 +99,9 @@ func newChainSupport( cs.StatusReporter = consensus.StaticStatusReporter{ClusterRelation: types.ClusterRelationNone, Status: types.StatusActive} } + clusterRelation, status := cs.StatusReporter.StatusReport() + registrar.ReportRelationAndStatusMetrics(cs.ChannelID(), clusterRelation, status) + logger.Debugf("[channel: %s] Done creating channel support resources", cs.ChannelID()) return cs, nil diff --git a/orderer/common/multichannel/metrics.go b/orderer/common/multichannel/metrics.go new file mode 100644 index 00000000000..b11236dc76e --- /dev/null +++ b/orderer/common/multichannel/metrics.go @@ -0,0 +1,92 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package multichannel + +import ( + "github.com/hyperledger/fabric/common/metrics" + "github.com/hyperledger/fabric/orderer/common/types" +) + +var ( + statusOpts = metrics.GaugeOpts{ + Namespace: "participation", + Subsystem: "", + Name: "status", + Help: "The channel participation status of the node: 0 if inactive, 1 if active, 2 if onboarding.", + LabelNames: []string{"channel"}, + StatsdFormat: "%{#fqname}.%{channel}", + } + + relationOpts = metrics.GaugeOpts{ + Namespace: "participation", + Subsystem: "", + Name: "cluster_relation", + Help: "The channel participation cluster relation of the node: 0 if none, 1 if consenter, 2 if follower, 3 if config-tracker.", + LabelNames: []string{"channel"}, + StatsdFormat: "%{#fqname}.%{channel}", + } +) + +// Metrics defines the metrics for the cluster. +type Metrics struct { + Status metrics.Gauge + Relation metrics.Gauge +} + +// A MetricsProvider is an abstraction for a metrics provider. It is a factory for +// Counter, Gauge, and Histogram meters. +type MetricsProvider interface { + // NewCounter creates a new instance of a Counter. + NewCounter(opts metrics.CounterOpts) metrics.Counter + // NewGauge creates a new instance of a Gauge. + NewGauge(opts metrics.GaugeOpts) metrics.Gauge + // NewHistogram creates a new instance of a Histogram. + NewHistogram(opts metrics.HistogramOpts) metrics.Histogram +} + +//go:generate mockery -dir . -name MetricsProvider -case underscore -output ./mocks/ + +// NewMetrics initializes new metrics for the cluster infrastructure. +func NewMetrics(m MetricsProvider) *Metrics { + return &Metrics{ + Status: m.NewGauge(statusOpts), + Relation: m.NewGauge(relationOpts), + } +} + +func (m *Metrics) reportStatus(channel string, status types.Status) { + var s int + switch status { + case types.StatusInactive: + s = 0 + case types.StatusActive: + s = 1 + case types.StatusOnBoarding: + s = 2 + default: + logger.Panicf("Programming error: unexpected status %s", status) + } + m.Status.With("channel", channel).Set(float64(s)) +} + +func (m *Metrics) reportRelation(channel string, relation types.ClusterRelation) { + var r int + switch relation { + case types.ClusterRelationNone: + r = 0 + case types.ClusterRelationConsenter: + r = 1 + case types.ClusterRelationFollower: + r = 2 + case types.ClusterRelationConfigTracker: + r = 3 + default: + logger.Panicf("Programming error: unexpected relation %s", relation) + + } + m.Relation.With("channel", channel).Set(float64(r)) +} diff --git a/orderer/common/multichannel/metrics_test.go b/orderer/common/multichannel/metrics_test.go new file mode 100644 index 00000000000..44ef44a22be --- /dev/null +++ b/orderer/common/multichannel/metrics_test.go @@ -0,0 +1,167 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package multichannel + +import ( + "github.com/hyperledger/fabric/common/metrics/metricsfakes" + "github.com/hyperledger/fabric/orderer/common/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Metrics", func() { + Context("NewMetrics", func() { + var ( + fakeProvider *metricsfakes.Provider + fakeGauge *metricsfakes.Gauge + ) + + BeforeEach(func() { + fakeProvider = &metricsfakes.Provider{} + fakeGauge = &metricsfakes.Gauge{} + + fakeProvider.NewGaugeReturns(fakeGauge) + }) + + It("uses the provider to initialize a new Metrics object", func() { + metrics := NewMetrics(fakeProvider) + + Expect(metrics).NotTo(BeNil()) + Expect(fakeProvider.NewGaugeCallCount()).To(Equal(2)) + + Expect(metrics.Relation).To(Equal(fakeGauge)) + Expect(metrics.Status).To(Equal(fakeGauge)) + }) + }) + + Context("reportStatus", func() { + var ( + fakeProvider *metricsfakes.Provider + fakeGauge *metricsfakes.Gauge + metrics *Metrics + ) + + BeforeEach(func() { + fakeProvider = &metricsfakes.Provider{} + fakeGauge = &metricsfakes.Gauge{} + + fakeProvider.NewGaugeReturns(fakeGauge) + fakeGauge.WithReturns(fakeGauge) + + metrics = NewMetrics(fakeProvider) + Expect(metrics.Status).To(Equal(fakeGauge)) + }) + + It("reports status inactive as a gauge", func() { + metrics.reportStatus("fake-channel", types.StatusInactive) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(0))) + }) + + It("reports status active as a gauge", func() { + metrics.reportStatus("fake-channel", types.StatusActive) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(1))) + }) + + It("reports status onboarding as a gauge", func() { + metrics.reportStatus("fake-channel", types.StatusOnBoarding) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(2))) + }) + + It("panics when reporting an unknown cluster status", func() { + Expect(func() { metrics.reportStatus("fake-channel", "unknown") }).To(Panic()) + }) + }) + + Context("reportRelation", func() { + var ( + fakeProvider *metricsfakes.Provider + fakeGauge *metricsfakes.Gauge + metrics *Metrics + ) + + BeforeEach(func() { + fakeProvider = &metricsfakes.Provider{} + fakeGauge = &metricsfakes.Gauge{} + + fakeProvider.NewGaugeReturns(fakeGauge) + fakeGauge.WithReturns(fakeGauge) + + metrics = NewMetrics(fakeProvider) + Expect(metrics.Relation).To(Equal(fakeGauge)) + }) + + It("reports relation none as a gauge", func() { + metrics.reportRelation("fake-channel", types.ClusterRelationNone) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(0))) + }) + + It("reports relation consenter as a gauge", func() { + metrics.reportRelation("fake-channel", types.ClusterRelationConsenter) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(1))) + }) + + It("reports relation follower as a gauge", func() { + metrics.reportRelation("fake-channel", types.ClusterRelationFollower) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(2))) + }) + + It("reports relation config-tracker as a gauge", func() { + metrics.reportRelation("fake-channel", types.ClusterRelationConfigTracker) + Expect(fakeGauge.WithCallCount()).To(Equal(1)) + Expect(fakeGauge.WithArgsForCall(0)).To(Equal([]string{"channel", "fake-channel"})) + Expect(fakeGauge.SetCallCount()).To(Equal(1)) + Expect(fakeGauge.SetArgsForCall(0)).To(Equal(float64(3))) + }) + + It("panics when reporting an unknown relation", func() { + Expect(func() { metrics.reportRelation("fake-channel", "unknown") }).To(Panic()) + }) + }) +}) + +func newFakeMetrics(fakeFields *fakeMetricsFields) *Metrics { + return &Metrics{ + Relation: fakeFields.fakeRelation, + Status: fakeFields.fakeStatus, + } +} + +type fakeMetricsFields struct { + fakeRelation *metricsfakes.Gauge + fakeStatus *metricsfakes.Gauge +} + +func newFakeMetricsFields() *fakeMetricsFields { + return &fakeMetricsFields{ + fakeRelation: newFakeGauge(), + fakeStatus: newFakeGauge(), + } +} + +func newFakeGauge() *metricsfakes.Gauge { + fakeGauge := &metricsfakes.Gauge{} + fakeGauge.WithReturns(fakeGauge) + return fakeGauge +} diff --git a/orderer/common/multichannel/mocks/metrics_provider.go b/orderer/common/multichannel/mocks/metrics_provider.go new file mode 100644 index 00000000000..8b0dc3cbb33 --- /dev/null +++ b/orderer/common/multichannel/mocks/metrics_provider.go @@ -0,0 +1,61 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + metrics "github.com/hyperledger/fabric/common/metrics" + mock "github.com/stretchr/testify/mock" +) + +// MetricsProvider is an autogenerated mock type for the MetricsProvider type +type MetricsProvider struct { + mock.Mock +} + +// NewCounter provides a mock function with given fields: opts +func (_m *MetricsProvider) NewCounter(opts metrics.CounterOpts) metrics.Counter { + ret := _m.Called(opts) + + var r0 metrics.Counter + if rf, ok := ret.Get(0).(func(metrics.CounterOpts) metrics.Counter); ok { + r0 = rf(opts) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(metrics.Counter) + } + } + + return r0 +} + +// NewGauge provides a mock function with given fields: opts +func (_m *MetricsProvider) NewGauge(opts metrics.GaugeOpts) metrics.Gauge { + ret := _m.Called(opts) + + var r0 metrics.Gauge + if rf, ok := ret.Get(0).(func(metrics.GaugeOpts) metrics.Gauge); ok { + r0 = rf(opts) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(metrics.Gauge) + } + } + + return r0 +} + +// NewHistogram provides a mock function with given fields: opts +func (_m *MetricsProvider) NewHistogram(opts metrics.HistogramOpts) metrics.Histogram { + ret := _m.Called(opts) + + var r0 metrics.Histogram + if rf, ok := ret.Get(0).(func(metrics.HistogramOpts) metrics.Histogram); ok { + r0 = rf(opts) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(metrics.Histogram) + } + } + + return r0 +} diff --git a/orderer/common/multichannel/multichannel_suite_test.go b/orderer/common/multichannel/multichannel_suite_test.go new file mode 100644 index 00000000000..e5737703854 --- /dev/null +++ b/orderer/common/multichannel/multichannel_suite_test.go @@ -0,0 +1,19 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package multichannel_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestMultichannel(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Multichannel Suite") +} diff --git a/orderer/common/multichannel/registrar.go b/orderer/common/multichannel/registrar.go index b5fcc6b6b7e..22a639dc6c3 100644 --- a/orderer/common/multichannel/registrar.go +++ b/orderer/common/multichannel/registrar.go @@ -55,14 +55,15 @@ type Registrar struct { systemChannelID string systemChannel *ChainSupport - consenters map[string]consensus.Consenter - ledgerFactory blockledger.Factory - signer identity.SignerSerializer - blockcutterMetrics *blockcutter.Metrics - templator msgprocessor.ChannelConfigTemplator - callbacks []channelconfig.BundleActor - bccsp bccsp.BCCSP - clusterDialer *cluster.PredicateDialer + consenters map[string]consensus.Consenter + ledgerFactory blockledger.Factory + signer identity.SignerSerializer + blockcutterMetrics *blockcutter.Metrics + templator msgprocessor.ChannelConfigTemplator + callbacks []channelconfig.BundleActor + bccsp bccsp.BCCSP + clusterDialer *cluster.PredicateDialer + channelParticipationMetrics *Metrics joinBlockFileRepo *filerepo.Repo } @@ -97,16 +98,17 @@ func NewRegistrar( clusterDialer *cluster.PredicateDialer, callbacks ...channelconfig.BundleActor) *Registrar { r := &Registrar{ - config: config, - chains: make(map[string]*ChainSupport), - followers: make(map[string]*follower.Chain), - pendingRemoval: make(map[string]bool), - ledgerFactory: ledgerFactory, - signer: signer, - blockcutterMetrics: blockcutter.NewMetrics(metricsProvider), - callbacks: callbacks, - bccsp: bccsp, - clusterDialer: clusterDialer, + config: config, + chains: make(map[string]*ChainSupport), + followers: make(map[string]*follower.Chain), + pendingRemoval: make(map[string]bool), + ledgerFactory: ledgerFactory, + signer: signer, + blockcutterMetrics: blockcutter.NewMetrics(metricsProvider), + callbacks: callbacks, + bccsp: bccsp, + clusterDialer: clusterDialer, + channelParticipationMetrics: NewMetrics(metricsProvider), } if config.ChannelParticipation.Enabled { @@ -854,18 +856,21 @@ func (r *Registrar) createFollower( blockPullerCreator, r, r.bccsp, + r, ) if err != nil { return nil, types.ChannelInfo{}, errors.WithMessagef(err, "failed to create follower for channel %s", channelID) } + clusterRelation, status := fChain.StatusReport() info := types.ChannelInfo{ - Name: channelID, - URL: "", - Height: ledgerRes.Height(), + Name: channelID, + URL: "", + Height: ledgerRes.Height(), + ClusterRelation: clusterRelation, + Status: status, } - info.ClusterRelation, info.Status = fChain.StatusReport() r.followers[channelID] = fChain @@ -1094,3 +1099,8 @@ func (r *Registrar) removeLedgerAsync(channelID string) { delete(r.pendingRemoval, channelID) }() } + +func (r *Registrar) ReportRelationAndStatusMetrics(channelID string, relation types.ClusterRelation, status types.Status) { + r.channelParticipationMetrics.reportRelation(channelID, relation) + r.channelParticipationMetrics.reportStatus(channelID, status) +} diff --git a/orderer/common/multichannel/registrar_test.go b/orderer/common/multichannel/registrar_test.go index c0b9a71ad67..f932c817bc7 100644 --- a/orderer/common/multichannel/registrar_test.go +++ b/orderer/common/multichannel/registrar_test.go @@ -1044,6 +1044,8 @@ func TestRegistrar_JoinChannel(t *testing.T) { consenter.IsChannelMemberReturns(true, nil) registrar := NewRegistrar(config, ledgerFactory, mockCrypto(), &disabled.Provider{}, cryptoProvider, nil) + fakeFields := newFakeMetricsFields() + registrar.channelParticipationMetrics = newFakeMetrics(fakeFields) t.Run("failure - consenter channel membership error", func(t *testing.T) { badConsenter := &mocks.Consenter{} @@ -1113,6 +1115,7 @@ func TestRegistrar_JoinChannel(t *testing.T) { joinBlockPath := filepath.Join(tmpdir, "pendingops", "joinblock", "my-raft-channel.joinblock") _, err = os.Stat(joinBlockPath) require.True(t, os.IsNotExist(err)) + checkMetrics(t, fakeFields, []string{"channel", "my-raft-channel"}, 1, 1, 1) }) }) @@ -1157,6 +1160,8 @@ func TestRegistrar_JoinChannel(t *testing.T) { consenter.IsChannelMemberReturns(false, nil) registrar := NewRegistrar(config, ledgerFactory, mockCrypto(), &disabled.Provider{}, cryptoProvider, dialer) + fakeFields := newFakeMetricsFields() + registrar.channelParticipationMetrics = newFakeMetrics(fakeFields) registrar.Initialize(mockConsenters) // Before join the chain, it doesn't exist @@ -1179,6 +1184,8 @@ func TestRegistrar_JoinChannel(t *testing.T) { fChain := registrar.GetFollower("my-raft-channel") require.NotNil(t, fChain) fChain.Halt() + + checkMetrics(t, fakeFields, []string{"channel", "my-raft-channel"}, 2, 2, 1) }) t.Run("Join app channel as follower then switch to member", func(t *testing.T) { @@ -1188,6 +1195,8 @@ func TestRegistrar_JoinChannel(t *testing.T) { consenter.IsChannelMemberReturns(false, nil) registrar := NewRegistrar(config, ledgerFactory, mockCrypto(), &disabled.Provider{}, cryptoProvider, dialer) + fakeFields := newFakeMetricsFields() + registrar.channelParticipationMetrics = newFakeMetrics(fakeFields) registrar.Initialize(mockConsenters) // Before join the chain, it doesn't exist @@ -1209,6 +1218,7 @@ func TestRegistrar_JoinChannel(t *testing.T) { joinBlockPath := filepath.Join(tmpdir, "pendingops", "joinblock", "my-raft-channel.joinblock") _, err = os.Stat(joinBlockPath) require.NoError(t, err) + checkMetrics(t, fakeFields, []string{"channel", "my-raft-channel"}, 2, 2, 1) // Let's assume the follower appended a block genesisBlockAppRaft.Header.Number = 0 @@ -1231,6 +1241,8 @@ func TestRegistrar_JoinChannel(t *testing.T) { // join-block removed after switching from follower to member _, err = os.Stat(joinBlockPath) require.True(t, os.IsNotExist(err)) + + checkMetrics(t, fakeFields, []string{"channel", "my-raft-channel"}, 1, 1, 2) }) t.Run("Join app channel as member, then switch to follower", func(t *testing.T) { @@ -1239,6 +1251,8 @@ func TestRegistrar_JoinChannel(t *testing.T) { consenter.IsChannelMemberReturns(true, nil) registrar := NewRegistrar(config, ledgerFactory, mockCrypto(), &disabled.Provider{}, cryptoProvider, dialer) + fakeFields := newFakeMetricsFields() + registrar.channelParticipationMetrics = newFakeMetrics(fakeFields) registrar.Initialize(mockConsenters) // Before join the chain, it doesn't exist @@ -1259,6 +1273,7 @@ func TestRegistrar_JoinChannel(t *testing.T) { require.Equal(t, 1, len(channelList.Channels)) require.Equal(t, "my-raft-channel", channelList.Channels[0].Name) require.Nil(t, channelList.SystemChannel) + checkMetrics(t, fakeFields, []string{"channel", "my-raft-channel"}, 1, 1, 1) // Let's assume the chain appended another config block genesisBlockAppRaft.Header.PreviousHash = protoutil.BlockHeaderHash(genesisBlockAppRaft.Header) @@ -1284,6 +1299,7 @@ func TestRegistrar_JoinChannel(t *testing.T) { require.Nil(t, channelList.SystemChannel) fChain.Halt() require.False(t, fChain.IsRunning()) + checkMetrics(t, fakeFields, []string{"channel", "my-raft-channel"}, 2, 1, 3) }) t.Run("Join system channel without on-boarding", func(t *testing.T) { @@ -1360,6 +1376,17 @@ func TestRegistrar_JoinChannel(t *testing.T) { }) } +func checkMetrics(t *testing.T, fakeFields *fakeMetricsFields, expectedLabels []string, expectedRelation, expectedStatus, expectedCallCount int) { + require.Equal(t, expectedCallCount, fakeFields.fakeRelation.SetCallCount()) + require.Equal(t, float64(expectedRelation), fakeFields.fakeRelation.SetArgsForCall(expectedCallCount-1)) + require.Equal(t, expectedCallCount, fakeFields.fakeRelation.WithCallCount()) + require.Equal(t, expectedLabels, fakeFields.fakeRelation.WithArgsForCall(expectedCallCount-1)) + require.Equal(t, expectedCallCount, fakeFields.fakeStatus.SetCallCount()) + require.Equal(t, float64(expectedStatus), fakeFields.fakeStatus.SetArgsForCall(expectedCallCount-1)) + require.Equal(t, expectedCallCount, fakeFields.fakeStatus.WithCallCount()) + require.Equal(t, expectedLabels, fakeFields.fakeStatus.WithArgsForCall(expectedCallCount-1)) +} + func TestRegistrar_RemoveChannel(t *testing.T) { var ( tmpdir string diff --git a/orderer/consensus/etcdraft/consenter.go b/orderer/consensus/etcdraft/consenter.go index 3018ef5bcf4..a73cd436469 100644 --- a/orderer/consensus/etcdraft/consenter.go +++ b/orderer/consensus/etcdraft/consenter.go @@ -25,6 +25,7 @@ import ( "github.com/hyperledger/fabric/internal/pkg/comm" "github.com/hyperledger/fabric/orderer/common/cluster" "github.com/hyperledger/fabric/orderer/common/localconfig" + "github.com/hyperledger/fabric/orderer/common/types" "github.com/hyperledger/fabric/orderer/consensus" "github.com/hyperledger/fabric/orderer/consensus/inactive" "github.com/hyperledger/fabric/protoutil" @@ -45,13 +46,14 @@ type InactiveChainRegistry interface { Stop() } -//go:generate mockery -dir . -name ChainManager -case underscore -output mocks +//go:generate counterfeiter -o mocks/chain_manager.go --fake-name ChainManager . ChainManager // ChainManager defines the methods from multichannel.Registrar needed by the Consenter. type ChainManager interface { GetConsensusChain(channelID string) consensus.Chain CreateChain(channelID string) SwitchChainToFollower(channelID string) + ReportRelationAndStatusMetrics(channelID string, relation types.ClusterRelation, status types.Status) } // Config contains etcdraft configurations @@ -160,10 +162,12 @@ func (c *Consenter) HandleChain(support consensus.ConsenterSupport, metadata *co id, err := c.detectSelfID(consenters) if err != nil { if c.InactiveChainRegistry != nil { - // There is a system channel, use the InactiveChainRegistry to track the future config updates of application channel. + // There is a system channel, use the InactiveChainRegistry to track the + // future config updates of application channel. c.InactiveChainRegistry.TrackChain(support.ChannelID(), support.Block(0), func() { c.ChainManager.CreateChain(support.ChannelID()) }) + c.ChainManager.ReportRelationAndStatusMetrics(support.ChannelID(), types.ClusterRelationConfigTracker, types.StatusInactive) return &inactive.Chain{Err: errors.Errorf("channel %s is not serviced by me", support.ChannelID())}, nil } @@ -234,6 +238,7 @@ func (c *Consenter) HandleChain(support consensus.ConsenterSupport, metadata *co c.Logger.Info("With system channel: after eviction InactiveChainRegistry.TrackChain will be called") haltCallback = func() { c.InactiveChainRegistry.TrackChain(support.ChannelID(), nil, func() { c.ChainManager.CreateChain(support.ChannelID()) }) + c.ChainManager.ReportRelationAndStatusMetrics(support.ChannelID(), types.ClusterRelationConfigTracker, types.StatusInactive) } } else { // when we do NOT have a system channel, we switch to a follower.Chain upon eviction. diff --git a/orderer/consensus/etcdraft/consenter_test.go b/orderer/consensus/etcdraft/consenter_test.go index 0b164d37e1b..d6789b30bbd 100644 --- a/orderer/consensus/etcdraft/consenter_test.go +++ b/orderer/consensus/etcdraft/consenter_test.go @@ -31,6 +31,8 @@ import ( "github.com/hyperledger/fabric/internal/pkg/comm" "github.com/hyperledger/fabric/orderer/common/cluster" clustermocks "github.com/hyperledger/fabric/orderer/common/cluster/mocks" + "github.com/hyperledger/fabric/orderer/common/types" + "github.com/hyperledger/fabric/orderer/consensus" "github.com/hyperledger/fabric/orderer/consensus/etcdraft" "github.com/hyperledger/fabric/orderer/consensus/etcdraft/mocks" "github.com/hyperledger/fabric/orderer/consensus/inactive" @@ -275,9 +277,16 @@ var _ = Describe("Consenter", func() { cryptoProvider, _ := sw.NewDefaultSecurityLevelWithKeystore(sw.NewDummyKeyStore()) chainInstance := &etcdraft.Chain{CryptoProvider: cryptoProvider} BeforeEach(func() { - chainManager.On("GetConsensusChain", "mychannel").Return(chainInstance) - chainManager.On("GetConsensusChain", "notmychannel").Return(nil) - chainManager.On("GetConsensusChain", "notraftchain").Return(&inactive.Chain{Err: errors.New("not a raft chain")}) + chainManager.GetConsensusChainStub = func(channel string) consensus.Chain { + switch channel { + case "mychannel": + return chainInstance + case "notraftchain": + return &inactive.Chain{Err: errors.New("not a raft chain")} + default: + return nil + } + } }) It("calls the chain manager and returns the reference when it is found", func() { consenter := newConsenter(chainManager, tlsCA.CertBytes(), certAsPEM) @@ -440,6 +449,11 @@ var _ = Describe("Consenter", func() { Expect(err).To(Not(HaveOccurred())) Expect(chain.Order(nil, 0).Error()).To(Equal("channel foo is not serviced by me")) Expect(consenter.icr.TrackChainCallCount()).To(Equal(1)) + Expect(chainManager.ReportRelationAndStatusMetricsCallCount()).To(Equal(1)) + channel, relation, status := chainManager.ReportRelationAndStatusMetricsArgsForCall(0) + Expect(channel).To(Equal("foo")) + Expect(relation).To(Equal(types.ClusterRelationConfigTracker)) + Expect(status).To(Equal(types.StatusInactive)) }) It("fails to handle chain if etcdraft options have not been provided", func() { diff --git a/orderer/consensus/etcdraft/mocks/chain_manager.go b/orderer/consensus/etcdraft/mocks/chain_manager.go index fcf26770c19..e8e58578e03 100644 --- a/orderer/consensus/etcdraft/mocks/chain_manager.go +++ b/orderer/consensus/etcdraft/mocks/chain_manager.go @@ -1,40 +1,230 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - +// Code generated by counterfeiter. DO NOT EDIT. package mocks import ( - consensus "github.com/hyperledger/fabric/orderer/consensus" + "sync" - mock "github.com/stretchr/testify/mock" + "github.com/hyperledger/fabric/orderer/common/types" + "github.com/hyperledger/fabric/orderer/consensus" + "github.com/hyperledger/fabric/orderer/consensus/etcdraft" ) -// ChainManager is an autogenerated mock type for the ChainManager type type ChainManager struct { - mock.Mock + CreateChainStub func(string) + createChainMutex sync.RWMutex + createChainArgsForCall []struct { + arg1 string + } + GetConsensusChainStub func(string) consensus.Chain + getConsensusChainMutex sync.RWMutex + getConsensusChainArgsForCall []struct { + arg1 string + } + getConsensusChainReturns struct { + result1 consensus.Chain + } + getConsensusChainReturnsOnCall map[int]struct { + result1 consensus.Chain + } + ReportRelationAndStatusMetricsStub func(string, types.ClusterRelation, types.Status) + reportRelationAndStatusMetricsMutex sync.RWMutex + reportRelationAndStatusMetricsArgsForCall []struct { + arg1 string + arg2 types.ClusterRelation + arg3 types.Status + } + SwitchChainToFollowerStub func(string) + switchChainToFollowerMutex sync.RWMutex + switchChainToFollowerArgsForCall []struct { + arg1 string + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *ChainManager) CreateChain(arg1 string) { + fake.createChainMutex.Lock() + fake.createChainArgsForCall = append(fake.createChainArgsForCall, struct { + arg1 string + }{arg1}) + fake.recordInvocation("CreateChain", []interface{}{arg1}) + fake.createChainMutex.Unlock() + if fake.CreateChainStub != nil { + fake.CreateChainStub(arg1) + } +} + +func (fake *ChainManager) CreateChainCallCount() int { + fake.createChainMutex.RLock() + defer fake.createChainMutex.RUnlock() + return len(fake.createChainArgsForCall) +} + +func (fake *ChainManager) CreateChainCalls(stub func(string)) { + fake.createChainMutex.Lock() + defer fake.createChainMutex.Unlock() + fake.CreateChainStub = stub +} + +func (fake *ChainManager) CreateChainArgsForCall(i int) string { + fake.createChainMutex.RLock() + defer fake.createChainMutex.RUnlock() + argsForCall := fake.createChainArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *ChainManager) GetConsensusChain(arg1 string) consensus.Chain { + fake.getConsensusChainMutex.Lock() + ret, specificReturn := fake.getConsensusChainReturnsOnCall[len(fake.getConsensusChainArgsForCall)] + fake.getConsensusChainArgsForCall = append(fake.getConsensusChainArgsForCall, struct { + arg1 string + }{arg1}) + fake.recordInvocation("GetConsensusChain", []interface{}{arg1}) + fake.getConsensusChainMutex.Unlock() + if fake.GetConsensusChainStub != nil { + return fake.GetConsensusChainStub(arg1) + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.getConsensusChainReturns + return fakeReturns.result1 +} + +func (fake *ChainManager) GetConsensusChainCallCount() int { + fake.getConsensusChainMutex.RLock() + defer fake.getConsensusChainMutex.RUnlock() + return len(fake.getConsensusChainArgsForCall) +} + +func (fake *ChainManager) GetConsensusChainCalls(stub func(string) consensus.Chain) { + fake.getConsensusChainMutex.Lock() + defer fake.getConsensusChainMutex.Unlock() + fake.GetConsensusChainStub = stub } -// CreateChain provides a mock function with given fields: channelID -func (_m *ChainManager) CreateChain(channelID string) { - _m.Called(channelID) +func (fake *ChainManager) GetConsensusChainArgsForCall(i int) string { + fake.getConsensusChainMutex.RLock() + defer fake.getConsensusChainMutex.RUnlock() + argsForCall := fake.getConsensusChainArgsForCall[i] + return argsForCall.arg1 } -// GetConsensusChain provides a mock function with given fields: channelID -func (_m *ChainManager) GetConsensusChain(channelID string) consensus.Chain { - ret := _m.Called(channelID) +func (fake *ChainManager) GetConsensusChainReturns(result1 consensus.Chain) { + fake.getConsensusChainMutex.Lock() + defer fake.getConsensusChainMutex.Unlock() + fake.GetConsensusChainStub = nil + fake.getConsensusChainReturns = struct { + result1 consensus.Chain + }{result1} +} + +func (fake *ChainManager) GetConsensusChainReturnsOnCall(i int, result1 consensus.Chain) { + fake.getConsensusChainMutex.Lock() + defer fake.getConsensusChainMutex.Unlock() + fake.GetConsensusChainStub = nil + if fake.getConsensusChainReturnsOnCall == nil { + fake.getConsensusChainReturnsOnCall = make(map[int]struct { + result1 consensus.Chain + }) + } + fake.getConsensusChainReturnsOnCall[i] = struct { + result1 consensus.Chain + }{result1} +} + +func (fake *ChainManager) ReportRelationAndStatusMetrics(arg1 string, arg2 types.ClusterRelation, arg3 types.Status) { + fake.reportRelationAndStatusMetricsMutex.Lock() + fake.reportRelationAndStatusMetricsArgsForCall = append(fake.reportRelationAndStatusMetricsArgsForCall, struct { + arg1 string + arg2 types.ClusterRelation + arg3 types.Status + }{arg1, arg2, arg3}) + fake.recordInvocation("ReportRelationAndStatusMetrics", []interface{}{arg1, arg2, arg3}) + fake.reportRelationAndStatusMetricsMutex.Unlock() + if fake.ReportRelationAndStatusMetricsStub != nil { + fake.ReportRelationAndStatusMetricsStub(arg1, arg2, arg3) + } +} + +func (fake *ChainManager) ReportRelationAndStatusMetricsCallCount() int { + fake.reportRelationAndStatusMetricsMutex.RLock() + defer fake.reportRelationAndStatusMetricsMutex.RUnlock() + return len(fake.reportRelationAndStatusMetricsArgsForCall) +} + +func (fake *ChainManager) ReportRelationAndStatusMetricsCalls(stub func(string, types.ClusterRelation, types.Status)) { + fake.reportRelationAndStatusMetricsMutex.Lock() + defer fake.reportRelationAndStatusMetricsMutex.Unlock() + fake.ReportRelationAndStatusMetricsStub = stub +} + +func (fake *ChainManager) ReportRelationAndStatusMetricsArgsForCall(i int) (string, types.ClusterRelation, types.Status) { + fake.reportRelationAndStatusMetricsMutex.RLock() + defer fake.reportRelationAndStatusMetricsMutex.RUnlock() + argsForCall := fake.reportRelationAndStatusMetricsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} - var r0 consensus.Chain - if rf, ok := ret.Get(0).(func(string) consensus.Chain); ok { - r0 = rf(channelID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(consensus.Chain) - } +func (fake *ChainManager) SwitchChainToFollower(arg1 string) { + fake.switchChainToFollowerMutex.Lock() + fake.switchChainToFollowerArgsForCall = append(fake.switchChainToFollowerArgsForCall, struct { + arg1 string + }{arg1}) + fake.recordInvocation("SwitchChainToFollower", []interface{}{arg1}) + fake.switchChainToFollowerMutex.Unlock() + if fake.SwitchChainToFollowerStub != nil { + fake.SwitchChainToFollowerStub(arg1) } +} + +func (fake *ChainManager) SwitchChainToFollowerCallCount() int { + fake.switchChainToFollowerMutex.RLock() + defer fake.switchChainToFollowerMutex.RUnlock() + return len(fake.switchChainToFollowerArgsForCall) +} + +func (fake *ChainManager) SwitchChainToFollowerCalls(stub func(string)) { + fake.switchChainToFollowerMutex.Lock() + defer fake.switchChainToFollowerMutex.Unlock() + fake.SwitchChainToFollowerStub = stub +} - return r0 +func (fake *ChainManager) SwitchChainToFollowerArgsForCall(i int) string { + fake.switchChainToFollowerMutex.RLock() + defer fake.switchChainToFollowerMutex.RUnlock() + argsForCall := fake.switchChainToFollowerArgsForCall[i] + return argsForCall.arg1 } -// SwitchChainToFollower provides a mock function with given fields: channelID -func (_m *ChainManager) SwitchChainToFollower(channelID string) { - _m.Called(channelID) +func (fake *ChainManager) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.createChainMutex.RLock() + defer fake.createChainMutex.RUnlock() + fake.getConsensusChainMutex.RLock() + defer fake.getConsensusChainMutex.RUnlock() + fake.reportRelationAndStatusMetricsMutex.RLock() + defer fake.reportRelationAndStatusMetricsMutex.RUnlock() + fake.switchChainToFollowerMutex.RLock() + defer fake.switchChainToFollowerMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *ChainManager) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) } + +var _ etcdraft.ChainManager = new(ChainManager)