Add WaitForSemiSyncRecoverySeconds config option

go/config/config.go

@@ -277,7 +277,8 @@ type Configuration struct {
 	MaxConcurrentReplicaOperations             int               // Maximum number of concurrent operations on replicas
 	EnforceExactSemiSyncReplicas               bool              // If true, semi-sync replicas will be enabled/disabled to match the wait count in the desired priority order; this applies to LockedSemiSyncMaster and MasterWithTooManySemiSyncReplicas
 	RecoverLockedSemiSyncMaster                bool              // If true, orchestrator will recover from a LockedSemiSync state by enabling semi-sync on replicas to match the wait count; this behavior can be overridden by EnforceExactSemiSyncReplicas
-	ReasonableLockedSemiSyncMasterSeconds      uint              // Time to evaluate the LockedSemiSyncHypothesis before triggering the LockedSemiSync analysis; falls back to ReasonableReplicationLagSeconds if not set
+	ReasonableLockedSemiSyncMasterSeconds      uint              // Time to evaluate the LockedSemiSyncHypothesis before triggering the LockedSemiSyncMaster analysis; defaults to ReasonableReplicationLagSeconds if not set
+	WaitForSemiSyncRecoverySeconds             uint              // Time to wait for a successful recovery of LockedSemiSyncMaster or MasterWithTooManySemiSyncReplicas before considering the recovery failed; defaults to InstancePollSeconds * 3 if not set
 }
 
 // ToJSONString will marshal this configuration as JSON
@@ -452,6 +453,7 @@ func newConfiguration() *Configuration {
 		EnforceExactSemiSyncReplicas:               false,
 		RecoverLockedSemiSyncMaster:                false,
 		ReasonableLockedSemiSyncMasterSeconds:      0,
+		WaitForSemiSyncRecoverySeconds:             0,
 	}
 }
 
@@ -617,6 +619,9 @@ func (this *Configuration) postReadAdjustments() error {
 	if this.ReasonableLockedSemiSyncMasterSeconds == 0 {
 		this.ReasonableLockedSemiSyncMasterSeconds = uint(this.ReasonableReplicationLagSeconds)
 	}
+	if this.WaitForSemiSyncRecoverySeconds == 0 {
+		this.WaitForSemiSyncRecoverySeconds = this.InstancePollSeconds * 3
+	}
 
 	return nil
 }

go/logic/topology_recovery.go

@@ -1548,7 +1548,7 @@ func recoverSemiSyncReplicas(topologyRecovery *TopologyRecovery, analysisEntry i
 	// Wait for replica count to match desired wait count (this is what triggers the recovery)
 	AuditTopologyRecovery(topologyRecovery, "semi-sync: waiting for desired state:")
 	success := false
-	for i := 0; i < 10; i++ {
+	for i := uint(0); i < config.Config.WaitForSemiSyncRecoverySeconds; i++ {
 		masterInstance, err = inst.ReadTopologyInstance(&analysisEntry.AnalyzedInstanceKey)
 		if err != nil {
 			return true, topologyRecovery, fmt.Errorf("error re-reading master instance: %s", err.Error())