Generate and Send the User Agent String with Business Metrics (#3729)

Author: muhammad-othman

Diff for: generator/.DevConfigs/a069861d-cd92-4d43-b45b-3aa11112a39b.json

@@ -0,0 +1,9 @@
+{
+  "core": {
+    "changeLogMessages": [
+      "Update the AWS SDK for .NET to include encoded metrics in the `User-Agent` header to track which features were used for a given request (for example, which retry behavior and how credentials were resolved)"
+    ],
+    "type": "patch",
+    "updateMinimum": true
+  }
+}

Diff for: generator/ServiceModels/s3/s3.customizations.json

@@ -8,7 +8,6 @@
             { "operation": "addAfter", "newType": "Amazon.S3.Internal.AmazonS3RedirectHandler", "targetType": "Amazon.Runtime.Internal.Unmarshaller" },
             { "operation": "addBefore", "newType": "Amazon.S3.Internal.S3Express.S3ExpressPreSigner", "targetType": "Amazon.Runtime.Internal.Signer" },
             { "operation": "addAfter", "newType": "Amazon.S3.Internal.AmazonS3PostMarshallHandler", "targetType": "Amazon.Runtime.Internal.EndpointResolver" },
-            { "operation": "addAfter", "newType": "Amazon.S3.Internal.AmazonS3UserAgentHandler", "targetType": "Amazon.Runtime.Internal.ChecksumHandler" },
             {
                 "condition":"this.Config.RetryMode == RequestRetryMode.Standard",
                 "operation": "replace",

Diff for: sdk/src/Core/Amazon.Runtime/AmazonWebServiceRequest.cs

@@ -14,7 +14,7 @@
  */
 using System;
 using System.Collections.Generic;
-using System.Text;
+using Amazon.Runtime.Internal.UserAgent;
 
 namespace Amazon.Runtime
 {
@@ -26,7 +26,7 @@ public abstract partial class AmazonWebServiceRequest : Amazon.Runtime.Internal.
         private readonly object _lock = new object();
 
         internal RequestEventHandler mBeforeRequestEvent;
-        internal string UserAgentAddition { get; set; } = null;
+        UserAgentDetails Amazon.Runtime.Internal.IAmazonWebServiceRequest.UserAgentDetails { get; } = new UserAgentDetails();
 
         internal event RequestEventHandler BeforeRequestEvent
         {

Diff for: sdk/src/Core/Amazon.Runtime/Internal/IAmazonWebServiceRequest.cs

@@ -1,4 +1,5 @@
-using System;
+using Amazon.Runtime.Internal.UserAgent;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -15,7 +16,7 @@ public interface IAmazonWebServiceRequest
 
         Dictionary<string, object> RequestState { get; }
 
-
         SignatureVersion SignatureVersion { get; set; }
+        UserAgentDetails UserAgentDetails { get; }
     }
 }

Diff for: sdk/src/Core/Amazon.Runtime/Internal/UserAgent/UserAgentDetails.cs

@@ -13,6 +13,7 @@
  * permissions and limitations under the License.
  */
 
+using System.Text;
 using System.Collections.Generic;
 
 namespace Amazon.Runtime.Internal.UserAgent
@@ -26,12 +27,25 @@ public class UserAgentDetails
     {
         private const int MaxSizeBytes = 1024; // 1 KB size limit
         private readonly HashSet<string> _trackedFeatureIds = new HashSet<string>();
+        private readonly StringBuilder _userAgentBuilder = new StringBuilder();
 
         /// <summary>
         /// Gets the list of tracked feature IDs.
         /// </summary>
         public IEnumerable<string> TrackedFeatureIds => _trackedFeatureIds;
 
+        /// <summary>
+        /// Adds a component to the user-agent string. 
+        /// </summary>
+        /// <param name="component">The user-agent component to append.</param>
+        public void AddUserAgentComponent(string component)
+        {
+            if (!string.IsNullOrEmpty(component))
+            {
+                _userAgentBuilder.Append(' ').Append(component);
+            }
+        }
+
         /// <summary>
         /// Adds a feature metric to be included in the User-Agent string.
         /// Duplicate entries are ignored.
@@ -43,17 +57,57 @@ public void AddFeature(UserAgentFeatureId featureId)
             _trackedFeatureIds.Add(featureId.Value);
         }
 
+        /// <summary>
+        /// Appends the metrics user-agent to the existing user-agent and returns the full string.
+        /// </summary>
+        /// <returns>The final user-agent string including metrics data.</returns>
+        public string GenerateUserAgentWithMetrics()
+        {
+            var metricsUserAgent = GenerateMetricsUserAgent();
+            if (!string.IsNullOrEmpty(metricsUserAgent))
+            {
+                return $"{_userAgentBuilder.ToString().Trim()} {metricsUserAgent}";
+            }
+            return _userAgentBuilder.ToString().Trim();
+        }
+
         /// <summary>
         /// Generates the User-Agent metrics string.
         /// Ensures the final string does not exceed 1024 bytes.
         /// </summary>
         /// <returns>The formatted User-Agent metrics string.</returns>
-#pragma warning disable CA1822 // Mark members as static
-        public string GenerateMetricsUserAgent()
-#pragma warning restore CA1822 // Mark members as static
+        private string GenerateMetricsUserAgent()
         {
-            // TODO
-            return "m/{metricsString}";
+            if (_trackedFeatureIds.Count == 0)
+            {
+                return string.Empty;
+            }
+            var metricsString = $"m/{string.Join(",", _trackedFeatureIds)}";
+            metricsString = TruncateToSize(metricsString);
+
+            return metricsString;
+        }
+
+        /// <summary>
+        /// Truncates a comma-separated string to ensure it fits within a given byte limit.
+        /// </summary>
+        /// <param name="input">The input string.</param>
+        /// <returns>A truncated version of the string within the byte limit.</returns>
+        private static string TruncateToSize(string input)
+        {
+            byte[] bytes = Encoding.UTF8.GetBytes(input);
+            if (bytes.Length <= MaxSizeBytes)
+            {
+                return input;
+            }
+
+            var cutOffIndex = MaxSizeBytes;
+            while (cutOffIndex > 0 && bytes[cutOffIndex - 1] != ',')
+            {
+                cutOffIndex--;
+            }
+
+            return Encoding.UTF8.GetString(bytes, 0, cutOffIndex - 1); // Remove last comma
         }
     }
 }

Diff for: sdk/src/Core/Amazon.Runtime/Pipeline/Contexts.cs

@@ -67,7 +67,7 @@ public interface IAsyncRequestContext : IRequestContext
     {
         AsyncCallback Callback { get; }
         object State { get; }
-    }    
+    }
 
     public interface IAsyncResponseContext : IResponseContext
     {
@@ -96,7 +96,7 @@ public class RequestContext : IRequestContext
         IDictionary<string, object> _contextAttributes;
 
         public RequestContext(bool enableMetric)
-            : this (enableMetric, null)
+            : this(enableMetric, null)
         {
         }
 
@@ -106,7 +106,6 @@ public RequestContext(bool enableMetrics, ISigner clientSigner)
             this.Metrics = new RequestMetrics();
             this.Metrics.IsEnabled = enableMetrics;
             this.InvocationId = Guid.NewGuid();
-            this.UserAgentDetails = new UserAgentDetails();
         }
 
         public IRequest Request { get; set; }
@@ -120,10 +119,10 @@ public RequestContext(bool enableMetrics, ISigner clientSigner)
         public AmazonWebServiceRequest OriginalRequest { get; set; }
         public IMarshaller<IRequest, AmazonWebServiceRequest> Marshaller { get; set; }
         public ResponseUnmarshaller Unmarshaller { get; set; }
-        public InvokeOptionsBase Options { get; set; }        
+        public InvokeOptionsBase Options { get; set; }
         public ISigner Signer { get; set; }
         public BaseIdentity Identity { get; set; }
-        public UserAgentDetails UserAgentDetails { get; }
+        public UserAgentDetails UserAgentDetails { get => ((IAmazonWebServiceRequest)OriginalRequest).UserAgentDetails; }
 
 #if AWS_ASYNC_API
         public System.Threading.CancellationToken CancellationToken { get; set; }
@@ -162,11 +161,11 @@ internal set
 
         public Guid InvocationId { get; private set; }
 
-        public IDictionary<string, object> ContextAttributes 
-        { 
+        public IDictionary<string, object> ContextAttributes
+        {
             get
             {
-                if(_contextAttributes == null)
+                if (_contextAttributes == null)
                 {
                     _contextAttributes = new Dictionary<string, object>();
                 }
@@ -180,7 +179,7 @@ public IDictionary<string, object> ContextAttributes
 
     public class AsyncRequestContext : RequestContext, IAsyncRequestContext
     {
-        public AsyncRequestContext(bool enableMetrics, ISigner clientSigner):
+        public AsyncRequestContext(bool enableMetrics, ISigner clientSigner) :
             base(enableMetrics, clientSigner)
         {
         }
@@ -191,7 +190,7 @@ public AsyncRequestContext(bool enableMetrics, ISigner clientSigner):
 
     public class ResponseContext : IResponseContext
     {
-        public AmazonWebServiceResponse Response { get; set; }        
+        public AmazonWebServiceResponse Response { get; set; }
         public IWebResponseData HttpResponse { get; set; }
     }
 

Diff for: sdk/src/Core/Amazon.Runtime/Pipeline/Handlers/BaseAuthResolverHandler.cs

@@ -76,7 +76,7 @@ protected void PreInvoke(IExecutionContext executionContext)
                     {
                         // We can use DefaultAWSCredentials if it was set by the user for these schemes.
                         executionContext.RequestContext.Identity = clientConfig.DefaultAWSCredentials;
-                        return;
+                        break;
                     }
 
                     if (scheme is BearerAuthScheme && clientConfig.AWSTokenProvider != null)
@@ -99,15 +99,15 @@ protected void PreInvoke(IExecutionContext executionContext)
 #endif
 
                         executionContext.RequestContext.Identity = token;
-                        return;
+                        break;
                     }
 
                     var identityResolver = scheme.GetIdentityResolver(clientConfig.IdentityResolverConfiguration);
                     executionContext.RequestContext.Identity = identityResolver.ResolveIdentity();
 
                     if (executionContext.RequestContext.Identity != null)
                     {
-                        return;
+                        break;
                     }
                 }
                 catch (Exception ex)

Diff for: sdk/src/Core/Amazon.Runtime/Pipeline/Handlers/Marshaller.cs

@@ -102,7 +102,7 @@ protected static void PreInvoke(IExecutionContext executionContext)
                 }
 
                 SetRecursionDetectionHeader(requestContext.Request.Headers);
-                SetUserAgentHeader(requestContext);
+                UpdateUserAgentDetails(requestContext);
             }
         }
 
@@ -124,51 +124,19 @@ private static void SetRecursionDetectionHeader(IDictionary<string, string> head
             }
         }
 
-        private static void SetUserAgentHeader(IRequestContext requestContext)
+        private static void UpdateUserAgentDetails(IRequestContext requestContext)
         {
             var sb = new StringBuilder(256);
 
-            sb.Append(InternalSDKUtils.ReplaceInvalidUserAgentCharacters(requestContext.ClientConfig.UserAgent));
+            requestContext.UserAgentDetails.AddUserAgentComponent(InternalSDKUtils.ReplaceInvalidUserAgentCharacters(requestContext.ClientConfig.UserAgent));
 
             var clientAppId = requestContext.ClientConfig.ClientAppId;
             if (!string.IsNullOrEmpty(clientAppId))
-                sb.Append(" app/").Append(InternalSDKUtils.ReplaceInvalidUserAgentCharacters(clientAppId));
+                requestContext.UserAgentDetails.AddUserAgentComponent($"app/{InternalSDKUtils.ReplaceInvalidUserAgentCharacters(clientAppId)}");
 
-            sb.Append(" cfg/retry-mode#").Append(ToUserAgentHeaderString(requestContext.ClientConfig.RetryMode));
+            requestContext.UserAgentDetails.AddUserAgentComponent($"md/{(requestContext.IsAsync ? "ClientAsync" : "ClientSync")}");
 
-            sb.Append(" md/").Append(requestContext.IsAsync ? "ClientAsync" : "ClientSync");
-
-            sb.Append(" cfg/init-coll#").Append(AWSConfigs.InitializeCollections ? '1' : '0');
-
-            var userAgentAddition = requestContext.OriginalRequest.UserAgentAddition;
-            if (!string.IsNullOrEmpty(userAgentAddition))
-            {
-                sb.Append(' ').Append(InternalSDKUtils.ReplaceInvalidUserAgentCharacters(userAgentAddition));
-            }
-
-            var userAgent = sb.ToString();
-
-            if (requestContext.ClientConfig.UseAlternateUserAgentHeader)
-            {
-                requestContext.Request.Headers[HeaderKeys.XAmzUserAgentHeader] = userAgent;
-            }
-            else
-            {
-                requestContext.Request.Headers[HeaderKeys.UserAgentHeader] = userAgent;
-            }
-        }
-
-        private static string ToUserAgentHeaderString(RequestRetryMode requestRetryMode)
-        {
-            switch (requestRetryMode)
-            {
-                case RequestRetryMode.Standard:
-                    return "standard";
-                case RequestRetryMode.Adaptive:
-                    return "adaptive";
-                default:
-                    return requestRetryMode.ToString().ToLowerInvariant();
-            }
+            requestContext.UserAgentDetails.AddUserAgentComponent($"cfg/init-coll#{(AWSConfigs.InitializeCollections ? '1' : '0')}");
         }
     }
 }

Diff for: sdk/src/Core/Amazon.Runtime/Pipeline/HttpHandler/HttpHandler.cs

@@ -66,6 +66,8 @@ public override void InvokeSync(IExecutionContext executionContext)
             try
             {
                 SetMetrics(executionContext.RequestContext);
+                SetUserAgentHeader(executionContext.RequestContext);
+
                 IRequest wrappedRequest = executionContext.RequestContext.Request;
                 httpRequest = CreateWebRequest(executionContext.RequestContext);
                 httpRequest.SetRequestHeaders(wrappedRequest.Headers);
@@ -183,6 +185,8 @@ public override async System.Threading.Tasks.Task<T> InvokeAsync<T>(IExecutionCo
             try
             {
                 SetMetrics(executionContext.RequestContext);
+                SetUserAgentHeader(executionContext.RequestContext);
+
                 IRequest wrappedRequest = executionContext.RequestContext.Request;
                 httpRequest = CreateWebRequest(executionContext.RequestContext);
                 httpRequest.SetRequestHeaders(wrappedRequest.Headers);
@@ -496,5 +500,20 @@ private static System.IO.Stream GetInputStream(IRequestContext requestContext, S
             }
             return originalStream;
         }
+
+        private void SetUserAgentHeader(IRequestContext requestContext)
+        {
+            var metricsUserAgent = requestContext.UserAgentDetails.GenerateUserAgentWithMetrics();
+            Logger.DebugFormat("User-Agent Header: {0}", metricsUserAgent);
+
+            if (requestContext.ClientConfig.UseAlternateUserAgentHeader)
+            {
+                requestContext.Request.Headers[HeaderKeys.XAmzUserAgentHeader] = metricsUserAgent;
+            }
+            else
+            {
+                requestContext.Request.Headers[HeaderKeys.UserAgentHeader] = metricsUserAgent;
+            }
+        }
     }
 }

Diff for: sdk/src/Core/Amazon.Runtime/_bcl+netstandard/PaginatorUtils.cs

@@ -1,14 +1,13 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
+using Amazon.Runtime.Internal;
+using Amazon.Runtime.Internal.UserAgent;
 
 namespace Amazon.Runtime
 {
     public static class PaginatorUtils
     {
         public static void SetUserAgentAdditionOnRequest(AmazonWebServiceRequest request)
         {
-            request.UserAgentAddition = $" ft/paginator";
+            ((IAmazonWebServiceRequest)request).UserAgentDetails.AddFeature(UserAgentFeatureId.PAGINATOR);
         }
     }
 }

Diff for: sdk/src/Services/Glacier/Custom/Transfer/ArchiveTransferManager.cs

@@ -170,8 +170,7 @@ internal void UserAgentRequestEventHandlerSync(object sender, RequestEventArgs a
                 WebServiceRequestEventArgs wsArgs = args as WebServiceRequestEventArgs;
                 if (wsArgs != null)
                 {
-                    string currentUserAgent = wsArgs.Headers[AWSSDKUtils.UserAgentHeader];
-                    wsArgs.Headers[AWSSDKUtils.UserAgentHeader] = currentUserAgent + " ft/ArchiveTransferManager md/" + this.operation;
+                    ((Runtime.Internal.IAmazonWebServiceRequest)wsArgs.Request).UserAgentDetails.AddUserAgentComponent("ft/ArchiveTransferManager md/" + this.operation);
                 }
             }
         }