provider: Lower retry threshold for DNS resolution failures

aws/awserr.go

@@ -8,13 +8,29 @@ import (
 	"github.com/hashicorp/terraform/helper/resource"
 )
 
+// Returns true if the error matches all these conditions:
+//  * err is of type awserr.Error
+//  * Error.Code() matches code
+//  * Error.Message() contains message
 func isAWSErr(err error, code string, message string) bool {
 	if err, ok := err.(awserr.Error); ok {
 		return err.Code() == code && strings.Contains(err.Message(), message)
 	}
 	return false
 }
 
+// Returns true if the error matches all these conditions:
+//  * err is of type awserr.Error
+//  * Error.Code() matches code
+//  * Error.Message() contains message
+//  * Error.OrigErr() contains origErrMessage
+func isAWSErrExtended(err error, code string, message string, origErrMessage string) bool {
+	if !isAWSErr(err, code, message) {
+		return false
+	}
+	return strings.Contains(err.(awserr.Error).OrigErr().Error(), origErrMessage)
+}
+
 func retryOnAwsCode(code string, f func() (interface{}, error)) (interface{}, error) {
 	var resp interface{}
 	err := resource.Retry(1*time.Minute, func() *resource.RetryError {

aws/config.go

@@ -351,6 +351,26 @@ func (c *Config) Client() (interface{}, error) {
 		sess = sess.Copy(&aws.Config{MaxRetries: aws.Int(c.MaxRetries)})
 	}
 
+	// Generally, we want to configure a lower retry theshold for networking issues
+	// as the session retry threshold is very high by default and can mask permanent
+	// networking failures, such as a non-existent service endpoint.
+	// MaxRetries will override this logic if it has a lower retry threshold.
+	// NOTE: This logic can be fooled by other request errors raising the retry count
+	//       before any networking error occurs
+	sess.Handlers.Retry.PushBack(func(r *request.Request) {
+		// We currently depend on the DefaultRetryer exponential backoff here.
+		// ~10 retries gives a fair backoff of a few seconds.
+		if r.RetryCount < 9 {
+			return
+		}
+		// RequestError: send request failed
+		// caused by: Post https://FQDN/: dial tcp: lookup FQDN: no such host
+		if isAWSErrExtended(r.Error, "RequestError", "send request failed", "no such host") {
+			log.Printf("[WARN] Disabling retries after next request due to networking issue")
+			r.Retryable = aws.Bool(false)
+		}
+	})
+
 	// This restriction should only be used for Route53 sessions.
 	// Other resources that have restrictions should allow the API to fail, rather
 	// than Terraform abstracting the region for the user. This can lead to breaking