metrics: Support the Summary metric type (#756)

Prometheus supports two distribution types: histogram and summary. Histograms track values over a set of fixed buckets. Summaries, on the other hand, track fixed quantiles over a sliding time window. This change introduces a Summary metric type that is backed by a ring buffer of hdrhistograms. These histograms may either auto-resize or have a fixed upper bound. The histograms are "rotated" so that old values expire after a maximum lifetime.
linkerd · Dec 2, 2020 · ee3fa14 · ee3fa14
1 parent c44027a
commit ee3fa14
Show file tree

Hide file tree

Showing 6 changed files with 505 additions and 22 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -54,6 +54,12 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034"
 
+[[package]]
+name = "arrayvec"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
+
 [[package]]
 name = "async-stream"
 version = "0.2.1"
@@ -240,6 +246,16 @@ dependencies = [
  "build_const",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
+dependencies = [
+ "crossbeam-utils",
+ "maybe-uninit",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.7.2"
@@ -308,6 +324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fac2277e84e5e858483756647a9d0aa8d9a2b7cba517fd84325a0aaa69a0909"
 dependencies = [
  "libc",
+ "miniz-sys",
  "miniz_oxide_c_api",
 ]
 
@@ -518,6 +535,20 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "hdrhistogram"
+version = "7.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3c22708574c44e924720c5b3a116326c688e6d532f438c77c007ec8768644f9"
+dependencies = [
+ "base64 0.12.3",
+ "byteorder",
+ "crossbeam-channel",
+ "flate2",
+ "nom 5.1.2",
+ "num-traits 0.2.6",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.0"
@@ -736,6 +767,19 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "lexical-core"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
+dependencies = [
+ "arrayvec",
+ "bitflags",
+ "cfg-if 0.1.10",
+ "ryu",
+ "static_assertions",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.76"
@@ -1160,10 +1204,13 @@ version = "0.1.0"
 dependencies = [
  "deflate",
  "futures 0.3.5",
+ "hdrhistogram",
  "http 0.2.1",
  "hyper",
  "indexmap",
+ "parking_lot",
  "quickcheck",
+ "tokio",
  "tracing",
 ]
 
@@ -1609,6 +1656,12 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376"
 
+[[package]]
+name = "maybe-uninit"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
+
 [[package]]
 name = "memchr"
 version = "0.1.11"
@@ -1633,6 +1686,16 @@ dependencies = [
  "libmimalloc-sys",
 ]
 
+[[package]]
+name = "miniz-sys"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e9e3ae51cea1576ceba0dde3d484d30e6e5b86dee0b2d412fe3a16a15c98202"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.1.2"
@@ -1742,6 +1805,17 @@ version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff"
 
+[[package]]
+name = "nom"
+version = "5.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
+dependencies = [
+ "lexical-core",
+ "memchr 2.3.3",
+ "version_check",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.39"
@@ -1908,7 +1982,7 @@ checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c"
 dependencies = [
  "byteorder",
  "libc",
- "nom",
+ "nom 2.2.1",
  "rustc_version",
 ]
 
@@ -2394,6 +2468,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "string"
 version = "0.2.0"
@@ -2913,6 +2993,12 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f2c54fe5e8d6907c60dc6fba532cc8529245d97ff4e26cb490cb462de114ba4"
 
+[[package]]
+name = "version_check"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
+
 [[package]]
 name = "want"
 version = "0.3.0"

diff --git a/linkerd/metrics/Cargo.toml b/linkerd/metrics/Cargo.toml
@@ -7,15 +7,20 @@ publish = false
 
 [features]
 default = []
+summary = ["hdrhistogram", "parking_lot", "tokio"]
 test_util = []
 
 [dependencies]
 deflate = { version = "0.7.18", features = ["gzip"] }
 futures = "0.3"
+hdrhistogram = { version = "7.1", optional = true }
 http = "0.2"
 hyper = "0.13.7"
 indexmap = "1.0"
+parking_lot = { version = "0.11", optional = true }
+tokio = { version = "0.2", features = ["time"], optional = true }
 tracing = "0.1.2"
 
 [dev-dependencies]
 quickcheck = { version = "0.9", default-features = false }
+tokio = { version = "0.2", features = ["macros", "test-util", "time"] }
diff --git a/linkerd/metrics/src/histogram.rs b/linkerd/metrics/src/histogram.rs
@@ -45,9 +45,6 @@ pub enum Bucket {
 #[derive(Debug)]
 pub struct Bounds(pub &'static [Bucket]);
 
-/// Helper that lazily formats metric keys as {0}_{1}.
-struct Key<A: fmt::Display, B: fmt::Display>(A, B);
-
 /// Helper that lazily formats an `{K}="{V}"`" label.
 struct Label<K: fmt::Display, V: fmt::Display>(K, V);
 
@@ -189,11 +186,10 @@ impl<V: Into<u64>, F: Factor> FmtMetric for Histogram<V, F> {
         let total = Counter::<F>::new();
         for (le, count) in self {
             total.add(count.into());
-            total.fmt_metric_labeled(f, Key(&name, "bucket"), Label("le", le))?;
+            total.fmt_metric_labeled(f, format_args!("{}_bucket", &name), Label("le", le))?;
         }
-        total.fmt_metric(f, Key(&name, "count"))?;
-        self.sum.fmt_metric(f, Key(&name, "sum"))?;
-
+        total.fmt_metric(f, format_args!("{}_count", &name))?;
+        self.sum.fmt_metric(f, format_args!("{}_sum", &name))?;
         Ok(())
     }
 
@@ -210,23 +206,19 @@ impl<V: Into<u64>, F: Factor> FmtMetric for Histogram<V, F> {
         let total = Counter::<F>::new();
         for (le, count) in self {
             total.add(count.into());
-            total.fmt_metric_labeled(f, Key(&name, "bucket"), (&labels, Label("le", le)))?;
+            total.fmt_metric_labeled(
+                f,
+                format_args!("{}_bucket", &name),
+                (&labels, Label("le", le)),
+            )?;
         }
-        total.fmt_metric_labeled(f, Key(&name, "count"), &labels)?;
-        self.sum.fmt_metric_labeled(f, Key(&name, "sum"), &labels)?;
-
+        total.fmt_metric_labeled(f, format_args!("{}_count", &name), &labels)?;
+        self.sum
+            .fmt_metric_labeled(f, format_args!("{}_sum", &name), &labels)?;
         Ok(())
     }
 }
 
-// ===== impl Key =====
-
-impl<A: fmt::Display, B: fmt::Display> fmt::Display for Key<A, B> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}_{}", self.0, self.1)
-    }
-}
-
 // ===== impl Label =====
 
 impl<K: fmt::Display, V: fmt::Display> FmtLabels for Label<K, V> {

diff --git a/linkerd/metrics/src/lib.rs b/linkerd/metrics/src/lib.rs
@@ -9,21 +9,25 @@ pub mod latency;
 mod prom;
 mod scopes;
 mod serve;
+#[cfg(feature = "summary")]
+mod summary;
 
 pub use self::counter::Counter;
 pub use self::gauge::Gauge;
 pub use self::histogram::Histogram;
 pub use self::prom::{FmtLabels, FmtMetric, FmtMetrics, Metric};
 pub use self::scopes::Scopes;
 pub use self::serve::Serve;
+#[cfg(feature = "summary")]
+pub use self::summary::Summary;
 
 #[macro_export]
 macro_rules! metrics {
     { $( $name:ident : $kind:ty { $help:expr } ),+ } => {
         $(
             #[allow(non_upper_case_globals)]
-            const $name: ::linkerd2_metrics::Metric<'static, &str, $kind> =
-                ::linkerd2_metrics::Metric {
+            const $name: $crate::Metric<'static, &str, $kind> =
+                $crate::Metric {
                     name: stringify!($name),
                     help: $help,
                     _p: ::std::marker::PhantomData,

diff --git a/linkerd/metrics/src/prom.rs b/linkerd/metrics/src/prom.rs
@@ -94,6 +94,16 @@ impl<'a, N: fmt::Display, M: FmtMetric> Metric<'a, N, M> {
         metric.fmt_metric(f, &self.name)
     }
 
+    /// Formats a single metric with labels.
+    pub fn fmt_metric_labeled<L: FmtLabels>(
+        &self,
+        f: &mut fmt::Formatter<'_>,
+        metric: &M,
+        labels: &L,
+    ) -> fmt::Result {
+        metric.fmt_metric_labeled(f, &self.name, labels)
+    }
+
     /// Formats a single metric across labeled scopes.
     pub fn fmt_scopes<'s, L, S: 's, I, F>(
         &self,