feat: add entity count distribution

caraml-dev · Oct 9, 2023 · 836a658 · 836a658
1 parent b762f98
commit 836a658
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 0 deletions.
diff --git a/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/MonitoringConfig.java b/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/MonitoringConfig.java
@@ -13,5 +13,8 @@
 public class MonitoringConfig {
   record TimerConfig(List<Double> percentiles, Integer minBucketMs, Integer maxBucketMs) {}
 
+  record EntityCountDistributionConfig(List<Double> percentiles, Integer maxExpectedCount) {}
+
   private TimerConfig timer;
+  private EntityCountDistributionConfig entityCountDistribution;
 }
diff --git a/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/ServingMetrics.java b/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/ServingMetrics.java
@@ -4,6 +4,7 @@
 import dev.caraml.store.protobuf.serving.ServingServiceProto.GetOnlineFeaturesRequest;
 import io.grpc.Status;
 import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.DistributionSummary;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Timer;
 import java.time.Duration;
@@ -61,6 +62,24 @@ private List<Counter> newEntityCounters(GetOnlineFeaturesRequest featureRequest)
         .toList();
   }
 
+  private List<DistributionSummary> newEntityCountHistograms(
+      GetOnlineFeaturesRequest featureRequest) {
+    return featureRequest.getFeaturesList().stream()
+        .map(FeatureReference::getFeatureTable)
+        .distinct()
+        .map(
+            table ->
+                DistributionSummary.builder("caraml_serving_entity_count_histogram")
+                    .tag("project", featureRequest.getProject())
+                    .tag("feature_table", table)
+                    .publishPercentiles(
+                        config.getEntityCountDistribution().percentiles().stream()
+                            .mapToDouble(Double::doubleValue)
+                            .toArray())
+                    .register(registry))
+        .toList();
+  }
+
   private Counter newKeyRetrievalCounter(String project) {
     return Counter.builder("caraml_serving_key_retrieval_count")
         .tag("project", project)
@@ -75,6 +94,8 @@ public void onRequestReceived(ReqT requestMessage) {
       newKeyRetrievalCounter(project).increment(featureRequest.getEntityRowsCount());
       newEntityCounters(featureRequest)
           .forEach(counter -> counter.increment(featureRequest.getEntityRowsCount()));
+      newEntityCountHistograms(featureRequest)
+          .forEach(histogram -> histogram.record(featureRequest.getEntityRowsCount()));
     }
   }
 

diff --git a/caraml-store-serving/src/main/resources/application.yaml b/caraml-store-serving/src/main/resources/application.yaml
@@ -23,6 +23,14 @@ caraml:
       minBucketMs: 1
       # Maximum expected latency
       maxBucketMs: 200
+    entityCountDistribution:
+      # Published percentiles
+      percentiles:
+        - 0.5
+        - 0.95
+        - 0.99
+      # Maximum expected entity count
+      maxExpectedCount: 150
 
   store:
     # Active store. Possible values: [redisCluster, redis, bigtable]