Handle NextRetryDelay option in workflow failures (#5946)

Workflows can now return an application error with NextRetryDelay option
to customize when the workflow will be retried again.
```
  return temporal.NewApplicationError(
    "some retryable error",
    "SomeType",
    temporal.ApplicationErrorOptions{NextRetryDelay: 2 * time.Minute},
  )
```

Currently Activity tasks can customize the next retry time. This is
bringing the same feature to workflow tasks as well.

Added unit tests.
Also added a test to assert next retry delay customization in
activities.

<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->

<!-- Have you made sure this change doesn't falsify anything currently
stated in `docs/`? If significant
new behavior is added, have you described that in `docs/`? -->

No

service/history/workflow/mutable_state_impl_test.go

@@ -1156,6 +1156,23 @@ func (s *mutableStateSuite) TestSpeculativeWorkflowTaskNotPersisted() {
 	}
 }
 
+func (s *mutableStateSuite) TestRetryWorkflowTask_WithNextRetryDelay() {
+	expectedDelayDuration := time.Minute
+	s.mutableState.executionInfo.HasRetryPolicy = true
+	applicationFailure := &failurepb.Failure{
+		Message: "application failure with customized next retry delay",
+		Source:  "application",
+		FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{
+			Type:           "application-failure-type",
+			NonRetryable:   false,
+			NextRetryDelay: durationpb.New(expectedDelayDuration),
+		}},
+	}
+
+	duration, retryState := s.mutableState.GetRetryBackoffDuration(applicationFailure)
+	s.Equal(enumspb.RETRY_STATE_IN_PROGRESS, retryState)
+	s.Equal(duration, expectedDelayDuration)
+}
 func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() {
 	s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes()
 

service/history/workflow/retry.go

@@ -75,6 +75,11 @@ func getBackoffInterval(
 		return backoff.NoBackoff, enumspb.RETRY_STATE_NON_RETRYABLE_FAILURE
 	}
 
+	// Check if the remote worker sent an application failure indicating a custom backoff duration.
+	delayedRetryDuration := nextRetryDelayFrom(failure)
+	if delayedRetryDuration != nil {
+		return nextBackoffInterval(now, currentAttempt, maxAttempts, initInterval, maxInterval, expirationTime, backoffCoefficient, makeBackoffAlgorithm(delayedRetryDuration))
+	}
 	return nextBackoffInterval(now, currentAttempt, maxAttempts, initInterval, maxInterval, expirationTime, backoffCoefficient, ExponentialBackoffAlgorithm)
 }