Add metrics on shard info (#980)

cadence-workflow · Jul 19, 2018 · aa5e689 · aa5e689
1 parent 17023c7
commit aa5e689
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 0 deletions.
diff --git a/common/metrics/defs.go b/common/metrics/defs.go
@@ -436,6 +436,8 @@ const (
 	ReplicatorTaskHistoryScope
 	// ReplicateHistoryEventsScope is the scope used by historyReplicator API for applying events
 	ReplicateHistoryEventsScope
+	// ShardInfoScope is the scope used when updating shard info
+	ShardInfoScope
 
 	NumHistoryScopes
 )
@@ -644,6 +646,7 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{
 		ReplicatorQueueProcessorScope:                {operation: "ReplicatorQueueProcessor"},
 		ReplicatorTaskHistoryScope:                   {operation: "ReplicatorTaskHistory"},
 		ReplicateHistoryEventsScope:                  {operation: "ReplicateHistoryEvents"},
+		ShardInfoScope:                               {operation: "ShardInfo"},
 	},
 	// Matching Scope Names
 	Matching: {
@@ -738,6 +741,8 @@ const (
 	ShardClosedCounter
 	ShardItemCreatedCounter
 	ShardItemRemovedCounter
+	ShardInfoTransferDiffTimer
+	ShardInfoTimerDiffTimer
 	MembershipChangedCounter
 	NumShardsGauge
 	GetEngineForShardErrorCounter
@@ -848,6 +853,8 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		ShardClosedCounter:                           {metricName: "shard-closed-count", metricType: Counter},
 		ShardItemCreatedCounter:                      {metricName: "sharditem-created-count", metricType: Counter},
 		ShardItemRemovedCounter:                      {metricName: "sharditem-removed-count", metricType: Counter},
+		ShardInfoTransferDiffTimer:                   {metricName: "shardinfo-transfer-diff", metricType: Timer},
+		ShardInfoTimerDiffTimer:                      {metricName: "shardinfo-timer-diff", metricType: Timer},
 		MembershipChangedCounter:                     {metricName: "membership-changed-count", metricType: Counter},
 		NumShardsGauge:                               {metricName: "numshards-gauge", metricType: Gauge},
 		GetEngineForShardErrorCounter:                {metricName: "get-engine-for-shard-errors", metricType: Counter},

diff --git a/service/history/shardContext.go b/service/history/shardContext.go
@@ -593,6 +593,7 @@ func (s *shardContextImpl) updateShardInfoLocked() error {
 		return nil
 	}
 	updatedShardInfo := copyShardInfo(s.shardInfo)
+	s.emitShardInfoMetricsLogsLocked()
 
 	err = s.shardManager.UpdateShard(&persistence.UpdateShardRequest{
 		ShardInfo:       updatedShardInfo,
@@ -611,6 +612,35 @@ func (s *shardContextImpl) updateShardInfoLocked() error {
 	return err
 }
 
+func (s *shardContextImpl) emitShardInfoMetricsLogsLocked() {
+	minTransferLevel := s.shardInfo.ClusterTransferAckLevel[s.currentCluster]
+	maxTransferLevel := s.shardInfo.ClusterTransferAckLevel[s.currentCluster]
+	for _, v := range s.shardInfo.ClusterTransferAckLevel {
+		if v < minTransferLevel {
+			minTransferLevel = v
+		}
+		if v > maxTransferLevel {
+			maxTransferLevel = v
+		}
+	}
+	diffTransferLevel := maxTransferLevel - minTransferLevel
+
+	minTimerLevel := s.shardInfo.ClusterTimerAckLevel[s.currentCluster]
+	maxTimerLevel := s.shardInfo.ClusterTimerAckLevel[s.currentCluster]
+	for _, v := range s.shardInfo.ClusterTimerAckLevel {
+		if v.Before(minTimerLevel) {
+			minTimerLevel = v
+		}
+		if v.After(maxTimerLevel) {
+			maxTimerLevel = v
+		}
+	}
+	diffTimerLevel := maxTimerLevel.Sub(minTimerLevel)
+
+	s.metricsClient.RecordTimer(metrics.ShardInfoScope, metrics.ShardInfoTransferDiffTimer, time.Duration(diffTransferLevel))
+	s.metricsClient.RecordTimer(metrics.ShardInfoScope, metrics.ShardInfoTimerDiffTimer, diffTimerLevel)
+}
+
 func (s *shardContextImpl) allocateTimerIDsLocked(timerTasks []persistence.Task) error {
 	// assign IDs for the timer tasks. They need to be assigned under shard lock.
 	for _, task := range timerTasks {