add time-to-first-byte for s3 (iopsystems#286)

Adds time-to-first-byte for s3
brayniac · Oct 8, 2024 · 5c05279 · 5c05279
1 parent 30d4dd2
commit 5c05279
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 3 deletions.
diff --git a/src/clients/store/s3/mod.rs b/src/clients/store/s3/mod.rs
@@ -2,7 +2,7 @@ use crate::workload::{ClientWorkItemKind, StoreClientRequest};
 use crate::*;
 
 use async_channel::Receiver;
-use bytes::Bytes;
+use bytes::{Bytes, BytesMut};
 use chrono::{DateTime, Utc};
 use http::{HeaderMap, Method, Version};
 use http_body_util::{BodyExt, Full};
@@ -149,11 +149,28 @@ async fn task(
                     let start = Instant::now();
 
                     match c.request(request).await {
-                        Ok(response) => {
+                        Ok(mut response) => {
                             let status = response.status().as_u16();
 
                             // wait until we have a complete response body
-                            let body = response.into_body().collect().await.unwrap().to_bytes();
+                            let mut body = BytesMut::new();
+
+                            let mut ttfb = None;
+
+                            while let Some(next) = response.frame().await {
+                                if let Ok(frame) = next {
+                                    if let Some(chunk) = frame.data_ref() {
+                                        if ttfb.is_none() {
+                                            ttfb = Some(start.elapsed());
+                                        }
+
+                                        body.extend_from_slice(chunk);
+                                    }
+                                } else {
+                                    STORE_RESPONSE_EX.increment();
+                                    STORE_GET_EX.increment();
+                                }
+                            }
 
                             let latency = start.elapsed();
 
@@ -168,6 +185,10 @@ async fn task(
 
                                     let _ =
                                         STORE_RESPONSE_LATENCY.increment(latency.as_nanos() as _);
+
+                                    if let Some(ttfb) = ttfb {
+                                        let _ = STORE_RESPONSE_TTFB.increment(ttfb.as_nanos() as _);
+                                    }
                                 }
                                 404 => {
                                     STORE_RESPONSE_OK.increment();
@@ -178,6 +199,11 @@ async fn task(
                                     let _ =
                                         STORE_RESPONSE_LATENCY.increment(latency.as_nanos() as _);
                                 }
+                                503 => {
+                                    STORE_RESPONSE_RATELIMITED.increment();
+                                    STORE_RESPONSE_EX.increment();
+                                    STORE_GET_EX.increment();
+                                }
                                 _ => {
                                     STORE_RESPONSE_EX.increment();
                                     STORE_GET_EX.increment();
@@ -227,6 +253,11 @@ async fn task(
                                     let _ =
                                         STORE_RESPONSE_LATENCY.increment(latency.as_nanos() as _);
                                 }
+                                503 => {
+                                    STORE_RESPONSE_RATELIMITED.increment();
+                                    STORE_RESPONSE_EX.increment();
+                                    STORE_PUT_EX.increment();
+                                }
                                 _ => {
                                     STORE_RESPONSE_EX.increment();
                                     STORE_PUT_EX.increment();
@@ -272,6 +303,11 @@ async fn task(
                                     let _ =
                                         STORE_RESPONSE_LATENCY.increment(latency.as_nanos() as _);
                                 }
+                                503 => {
+                                    STORE_RESPONSE_RATELIMITED.increment();
+                                    STORE_RESPONSE_EX.increment();
+                                    STORE_DELETE_EX.increment();
+                                }
                                 _ => {
                                     STORE_RESPONSE_EX.increment();
                                     STORE_DELETE_EX.increment();

diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs
@@ -631,6 +631,14 @@ counter!(PUBSUB_RECEIVE_OK, "subscriber/receive/ok");
 pub static STORE_RESPONSE_LATENCY: AtomicHistogram = AtomicHistogram::new(7, 64);
 pub static STORE_RESPONSE_LATENCY_HISTOGRAM: &str = "store_response_latency";
 
+#[metric(
+    name = STORE_RESPONSE_TTFB_HISTOGRAM,
+    description = "Distribution of storage client response time-to-first-byte for read operations",
+    metadata = { unit = "nanoseconds" }
+)]
+pub static STORE_RESPONSE_TTFB: AtomicHistogram = AtomicHistogram::new(7, 64);
+pub static STORE_RESPONSE_TTFB_HISTOGRAM: &str = "store_response_ttfb";
+
 counter!(STORE_CONNECT, "store_client/connect/total");
 counter!(STORE_CONNECT_OK, "store_client/connect/ok");
 counter!(STORE_CONNECT_EX, "store_client/connect/exception");