From 80ef0fabdf4f7cacf5dfb1f269b0ac57658d32f9 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 5 Mar 2020 10:13:21 +0800 Subject: [PATCH] Send additional 25th and 99th percentile for feature value metrics --- .../feast/ingestion/options/ImportOptions.java | 12 ++++++------ .../metrics/WriteFeatureValueMetricsDoFn.java | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java index c1bdcd5fd1..1fa127d662 100644 --- a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java +++ b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java @@ -65,8 +65,7 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, */ void setDeadLetterTableSpec(String deadLetterTableSpec); - // TODO: expound - @Description("MetricsAccumulator exporter type to instantiate.") + @Description("MetricsAccumulator exporter type to instantiate. Supported type: statsd") @Default.String("none") String getMetricsExporterType(); @@ -86,10 +85,11 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, void setStatsdPort(int StatsdPort); @Description( - "Fixed window size in seconds (default 30) to apply before aggregation of numerical value of features" - + "and writing the aggregated value to StatsD. Refer to feast.ingestion.transform.metrics.WriteFeatureValueMetricsDoFn" - + "for details on the metric names and types.") - @Default.Integer(30) + "Fixed window size in seconds (default 60) to apply before aggregating the numerical value of " + + "features and exporting the aggregated values as metrics. Refer to " + + "feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java" + + "for the metric nameas and types used.") + @Default.Integer(60) int getWindowSizeInSecForFeatureValueMetric(); void setWindowSizeInSecForFeatureValueMetric(int seconds); diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java index 8574d2414c..a4ed07b505 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java @@ -90,9 +90,11 @@ abstract static class Builder { public static String GAUGE_NAME_FEATURE_VALUE_MIN = "feature_value_min"; public static String GAUGE_NAME_FEATURE_VALUE_MAX = "feature_value_max"; public static String GAUGE_NAME_FEATURE_VALUE_MEAN = "feature_value_mean"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25 = "feature_value_percentile_25"; public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50 = "feature_value_percentile_50"; public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90 = "feature_value_percentile_90"; public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95 = "feature_value_percentile_95"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99 = "feature_value_percentile_99"; @Setup public void setup() { @@ -205,6 +207,12 @@ public void processElement( values[i] = valueList.get(i); } + double p25 = new Percentile().evaluate(values, 25); + if (p25 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, p25, tags); + double p50 = new Percentile().evaluate(values, 50); if (p50 < 0) { statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50, 0, tags); @@ -222,6 +230,12 @@ public void processElement( statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, 0, tags); } statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, p95, tags); + + double p99 = new Percentile().evaluate(values, 99); + if (p99 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, p99, tags); } }