Improve client tracing spans (#2044)

* Emit spans for implementers of map request middleware traits
* Instrument dispatch with its own span
* Fix trace span hierarchy
* Partially flatten the middleware span hierarchy
* Make `MapRequest::name` required
* Add sub-spans to the `load_response` span

Author: jdisanti

CHANGELOG.next.toml

@@ -677,3 +677,15 @@ The Unit type for a Union member is no longer rendered. The serializers and pars
 references = ["smithy-rs#1989"]
 meta = { "breaking" = true, "tada" = false, "bug" = false, "target" = "all" }
 author = "ysaito1001"
+
+[[aws-sdk-rust]]
+message = "Fixed and improved the request `tracing` span hierarchy to improve log messages, profiling, and debuggability."
+references = ["smithy-rs#2044", "smithy-rs#371"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "jdisanti"
+
+[[smithy-rs]]
+message = "Fixed and improved the request `tracing` span hierarchy to improve log messages, profiling, and debuggability."
+references = ["smithy-rs#2044", "smithy-rs#371"]
+meta = { "breaking" = false, "tada" = true, "bug" = false, "target" = "client"}
+author = "jdisanti"

aws/rust-runtime/aws-config/src/imds/client/token.rs

@@ -160,6 +160,10 @@ impl AsyncMapRequest for TokenMiddleware {
     type Error = ImdsError;
     type Future = Pin<Box<dyn Future<Output = Result<Request, Self::Error>> + Send + 'static>>;
 
+    fn name(&self) -> &'static str {
+        "attach_imds_token"
+    }
+
     fn apply(&self, request: Request) -> Self::Future {
         let this = self.clone();
         Box::pin(async move { this.add_token(request).await })

aws/rust-runtime/aws-config/src/meta/credentials/lazy_caching.rs

@@ -84,10 +84,10 @@ impl ProvideCredentials for LazyCachingCredentialsProvider {
                 // There may be other threads also loading simultaneously, but this is OK
                 // since the futures are not eagerly executed, and the cache will only run one
                 // of them.
-                let span = trace_span!("lazy_load_credentials");
                 let future = Timeout::new(loader.provide_credentials(), timeout_future);
                 cache
                     .get_or_load(|| {
+                        let span = trace_span!("lazy_load_credentials");
                         async move {
                             let credentials = future.await.map_err(|_err| {
                                 CredentialsError::provider_timed_out(load_timeout)

aws/rust-runtime/aws-endpoint/src/lib.rs

@@ -133,6 +133,10 @@ impl From<AwsAuthStageErrorKind> for AwsAuthStageError {
 impl MapRequest for AwsAuthStage {
     type Error = AwsAuthStageError;
 
+    fn name(&self) -> &'static str {
+        "resolve_endpoint"
+    }
+
     fn apply(&self, request: Request) -> Result<Request, Self::Error> {
         request.augment(|http_req, props| {
             let endpoint = props

aws/rust-runtime/aws-http/src/auth.rs

@@ -101,6 +101,10 @@ impl AsyncMapRequest for CredentialsStage {
     type Error = CredentialsStageError;
     type Future = Pin<Box<dyn Future<Output = Result<Request, Self::Error>> + Send + 'static>>;
 
+    fn name(&self) -> &'static str {
+        "retrieve_credentials"
+    }
+
     fn apply(&self, request: Request) -> BoxFuture<Result<Request, Self::Error>> {
         Box::pin(Self::load_creds(request))
     }

aws/rust-runtime/aws-http/src/recursion_detection.rs

@@ -33,6 +33,10 @@ impl RecursionDetectionStage {
 impl MapRequest for RecursionDetectionStage {
     type Error = std::convert::Infallible;
 
+    fn name(&self) -> &'static str {
+        "recursion_detection"
+    }
+
     fn apply(&self, request: Request) -> Result<Request, Self::Error> {
         request.augment(|mut req, _conf| {
             augument_request(&mut req, &self.env);

aws/rust-runtime/aws-http/src/user_agent.rs

@@ -582,6 +582,10 @@ lazy_static::lazy_static! {
 impl MapRequest for UserAgentStage {
     type Error = UserAgentStageError;
 
+    fn name(&self) -> &'static str {
+        "generate_user_agent"
+    }
+
     fn apply(&self, request: Request) -> Result<Request, Self::Error> {
         request.augment(|mut req, conf| {
             let ua = conf

aws/rust-runtime/aws-sig-auth/src/middleware.rs

@@ -155,6 +155,10 @@ fn signing_config(
 impl MapRequest for SigV4SigningStage {
     type Error = SigningStageError;
 
+    fn name(&self) -> &'static str {
+        "sigv4_sign_request"
+    }
+
     fn apply(&self, req: Request) -> Result<Request, Self::Error> {
         req.augment(|mut req, config| {
             let operation_config = config

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsFluentClientDecorator.kt

@@ -323,7 +323,7 @@ private fun renderCustomizableOperationSendMethod(
             /// Sends this operation's request
             pub async fn send<T, E>(self) -> Result<T, SdkError<E>>
             where
-                E: std::error::Error + 'static,
+                E: std::error::Error + Send + Sync + 'static,
                 O: #{ParseHttpResponse}<Output = Result<T, E>> + Send + Sync + Clone + 'static,
                 Retry: #{ClassifyRetry}<#{SdkSuccess}<T>, SdkError<E>> + Send + Sync + Clone,
             {

codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/client/CustomizableOperationGenerator.kt

@@ -160,7 +160,7 @@ class CustomizableOperationGenerator(
                 /// Sends this operation's request
                 pub async fn send<T, E>(self) -> Result<T, SdkError<E>>
                 where
-                    E: std::error::Error + 'static,
+                    E: std::error::Error + Send + Sync + 'static,
                     O: #{ParseHttpResponse}<Output = Result<T, E>> + Send + Sync + Clone + 'static,
                     Retry: Send + Sync + Clone,
                     <R as #{NewRequestPolicy}>::Policy: #{SmithyRetryPolicy}<O, T, E, Retry> + Clone,