Bugfix: when buffering events, the last write version should remain t…

…he same (#982)

* Bugfix: when buffering events, the last write version should remain the same

* Emit warn log if timer / transfer level diff exceed threshold

service/history/shardContext.go

@@ -103,6 +103,11 @@ type (
 
 var _ ShardContext = (*shardContextImpl)(nil)
 
+const (
+	logWarnTransferLevelDiff = 10000000 // 10 million
+	logWarnTimerLevelDiff    = time.Duration(30 * time.Minute)
+)
+
 func (s *shardContextImpl) GetShardID() int {
 	return s.shardID
 }
@@ -637,6 +642,13 @@ func (s *shardContextImpl) emitShardInfoMetricsLogsLocked() {
 	}
 	diffTimerLevel := maxTimerLevel.Sub(minTimerLevel)
 
+	if logWarnTransferLevelDiff < diffTransferLevel {
+		s.logger.Warn("Transfer level diff exceeds warn threshold.")
+	}
+	if logWarnTimerLevelDiff < diffTimerLevel {
+		s.logger.Warn("Timer level diff exceeds warn threshold.")
+	}
+
 	s.metricsClient.RecordTimer(metrics.ShardInfoScope, metrics.ShardInfoTransferDiffTimer, time.Duration(diffTransferLevel))
 	s.metricsClient.RecordTimer(metrics.ShardInfoScope, metrics.ShardInfoTimerDiffTimer, diffTimerLevel)
 }

service/history/workflowExecutionContext.go

@@ -208,23 +208,23 @@ func (c *workflowExecutionContext) updateHelper(builder *historyBuilder, transfe
 		return err
 	}
 
+	// Replicator passes in a custom builder as it already has the events
+	if builder == nil {
+		// If no builder is passed in then use the one as part of the updates
+		builder = updates.newEventsBuilder
+	}
+	executionInfo := c.msBuilder.GetExecutionInfo()
+	hasNewHistoryEvents := len(builder.history) > 0
+
 	// Replication state should only be updated after the UpdateSession is closed.  IDs for certain events are only
 	// generated on CloseSession as they could be buffered events.  The value for NextEventID will be wrong on
 	// mutable state if read before flushing the buffered events.
 	crossDCEnabled := c.msBuilder.GetReplicationState() != nil
-	if crossDCEnabled {
+	if crossDCEnabled && hasNewHistoryEvents {
 		lastEventID := c.msBuilder.GetNextEventID() - 1
 		c.msBuilder.UpdateReplicationStateLastEventID(sourceCluster, lastWriteVersion, lastEventID)
 	}
 
-	// Replicator passes in a custom builder as it already has the events
-	if builder == nil {
-		// If no builder is passed in then use the one as part of the updates
-		builder = updates.newEventsBuilder
-	}
-	executionInfo := c.msBuilder.GetExecutionInfo()
-
-	hasNewHistoryEvents := builder.history != nil && len(builder.history) > 0
 	// Some operations only update the mutable state. For example RecordActivityTaskHeartbeat.
 	if hasNewHistoryEvents {
 		firstEvent := builder.GetFirstEvent()