Change refresh window to 5-10 minutes. Adjust refresh strategy to immediately refresh the firs time an expired IMDS cred is encountered.

Author: ppittle

sdk/src/Core/Amazon.Runtime/Credentials/DefaultInstanceProfileAWSCredentials.cs

@@ -41,9 +41,15 @@ internal class DefaultInstanceProfileAWSCredentials : AWSCredentials, IDisposabl
         private const string FailedToGetCredentialsMessage = "Failed to retrieve credentials from EC2 Instance Metadata Service.";
         private static readonly TimeSpan _credentialsLockTimeout = TimeSpan.FromSeconds(5);
 
+        /// <summary>
+        /// Control flag: in the event IMDS returns an expired credential, a refresh must be immediately
+        /// retried, if it continues to fail, then retry every 5-10 minutes.
+        /// </summary>
+        private static volatile bool _imdsRefreshFailed = false;
+
         private const string _usingExpiredCredentialsFromIMDS =
             "Attempting credential expiration extension due to a credential service availability issue. " +
-            "A refresh of these credentials will be attempted again in 5-15 minutes.";
+            "A refresh of these credentials will be attempted again in 5-10 minutes.";
 
         private static DefaultInstanceProfileAWSCredentials _instance;
 
@@ -94,12 +100,24 @@ public override ImmutableCredentials GetCredentials()
                 {
                     if (null != _lastRetrievedCredentials)
                     {
+                        if (_lastRetrievedCredentials.IsExpiredWithin(TimeSpan.Zero) &&
+                            !_imdsRefreshFailed)
+                        {
+                            // this is the first failure - immediately try to renew
+                            _imdsRefreshFailed = true;
+                            _lastRetrievedCredentials = FetchCredentials();
+                        }
+
                         // if credentials are expired, we'll still return them, but log a message about
                         // them being expired.
                         if (_lastRetrievedCredentials.IsExpiredWithin(TimeSpan.Zero))
                         {
                             _logger.InfoFormat(_usingExpiredCredentialsFromIMDS);
                         }
+                        else
+                        {
+                            _imdsRefreshFailed = false;
+                        }
 
                         return _lastRetrievedCredentials?.Credentials.Copy();
                     }
@@ -121,12 +139,24 @@ public override ImmutableCredentials GetCredentials()
                         _lastRetrievedCredentials = FetchCredentials();
                     }
 
+                    if (_lastRetrievedCredentials.IsExpiredWithin(TimeSpan.Zero) && 
+                         !_imdsRefreshFailed)
+                    {
+                        // this is the first failure - immediately try to renew
+                        _imdsRefreshFailed = true;
+                        _lastRetrievedCredentials = FetchCredentials();
+                    }
+                    
                     // if credentials are expired, we'll still return them, but log a message about
                     // them being expired.
                     if (_lastRetrievedCredentials.IsExpiredWithin(TimeSpan.Zero))
                     {
                         _logger.InfoFormat(_usingExpiredCredentialsFromIMDS);
                     }
+                    else
+                    {
+                        _imdsRefreshFailed = false;
+                    }
 
                     credentials = _lastRetrievedCredentials.Credentials?.Copy();
                 }
@@ -169,10 +199,24 @@ private void RenewCredentials(object unused)
                 // would remain unchanged and would continue to be returned in GetCredentials()
                 _lastRetrievedCredentials = FetchCredentials();
 
+                // check for a first time failure
+                if (!_imdsRefreshFailed &&
+                    _lastRetrievedCredentials.IsExpiredWithin(TimeSpan.Zero))
+                {
+                    // this is the first failure - immediately try to renew
+                    _imdsRefreshFailed = true;
+                    _lastRetrievedCredentials = FetchCredentials();
+                }
+
+                // first failure refresh failed OR subsequent refresh failed.
                 if (_lastRetrievedCredentials.IsExpiredWithin(TimeSpan.Zero))
                 {
                     // relax the refresh rate to at least 5 minutes
-                    refreshRate = TimeSpan.FromMinutes(new Random().Next(5, 16));
+                    refreshRate = TimeSpan.FromMinutes(new Random().Next(5, 11));
+                }
+                else
+                {
+                    _imdsRefreshFailed = false;
                 }
             }
             catch (OperationCanceledException e)

sdk/src/Core/Amazon.Runtime/Credentials/InstanceProfileAWSCredentials.cs

@@ -44,7 +44,7 @@ public class InstanceProfileAWSCredentials : URIBasedRefreshingCredentialHelper
 
         private const string _receivedExpiredCredentialsFromIMDS =
             "Attempting credential expiration extension due to a credential service availability issue. " +
-            "A refresh of these credentials will be attempted again in 5-15 minutes.";
+            "A refresh of these credentials will be attempted again in 5-10 minutes.";
 
         private Logger _logger;
 
@@ -107,7 +107,7 @@ protected override CredentialsRefreshState GenerateNewCredentials()
                 // use a custom refresh time
 
                 #pragma warning disable CS0612 // Type or member is obsolete
-                var newExpiryTime = AWSSDKUtils.CorrectedUtcNow.ToLocalTime() + TimeSpan.FromMinutes(new Random().Next(5, 16));
+                var newExpiryTime = AWSSDKUtils.CorrectedUtcNow.ToLocalTime() + TimeSpan.FromMinutes(new Random().Next(5, 11));
                 #pragma warning restore CS0612 // Type or member is obsolete
 
                 _currentRefreshState = new CredentialsRefreshState(newState.Credentials.Copy(), newExpiryTime);