From a89c47a402741559860e0704e2ba7c9ae4b49b06 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Mon, 17 May 2021 17:54:26 +0000 Subject: [PATCH] fix(log_to_metric transform): Drop events where `field` is null This is the documented behavior but we were not honoring it. Signed-off-by: Jesse Szwedko --- src/internal_events/log_to_metric.rs | 20 ++++++ src/transforms/log_to_metric.rs | 101 ++++++++++++++++++--------- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/src/internal_events/log_to_metric.rs b/src/internal_events/log_to_metric.rs index bd95ed3d8017b..c3a75e049e018 100644 --- a/src/internal_events/log_to_metric.rs +++ b/src/internal_events/log_to_metric.rs @@ -3,6 +3,26 @@ use crate::template::TemplateParseError; use metrics::counter; use std::num::ParseFloatError; +pub(crate) struct LogToMetricFieldNull<'a> { + pub field: &'a str, +} + +impl<'a> InternalEvent for LogToMetricFieldNull<'a> { + fn emit_logs(&self) { + warn!( + message = "Field is null.", + null_field = %self.field, + internal_log_rate_secs = 30 + ); + } + + fn emit_metrics(&self) { + counter!("processing_errors_total", 1, + "error_type" => "field_null", + ); + } +} + pub(crate) struct LogToMetricFieldNotFound<'a> { pub field: &'a str, } diff --git a/src/transforms/log_to_metric.rs b/src/transforms/log_to_metric.rs index a18ca56f1d8d2..be2d4026cc30c 100644 --- a/src/transforms/log_to_metric.rs +++ b/src/transforms/log_to_metric.rs @@ -3,10 +3,10 @@ use crate::{ log_schema, DataType, GenerateConfig, GlobalOptions, TransformConfig, TransformDescription, }, event::metric::{Metric, MetricKind, MetricValue, StatisticKind}, - event::{Event, LogEvent, Value}, + event::{Event, Value}, internal_events::{ - LogToMetricFieldNotFound, LogToMetricParseFloatError, LogToMetricTemplateParseError, - TemplateRenderingFailed, + LogToMetricFieldNotFound, LogToMetricFieldNull, LogToMetricParseFloatError, + LogToMetricTemplateParseError, TemplateRenderingFailed, }, template::{Template, TemplateParseError, TemplateRenderingError}, transforms::{FunctionTransform, Transform}, @@ -75,6 +75,18 @@ pub enum MetricConfig { Summary(SummaryConfig), } +impl MetricConfig { + fn field(&self) -> &str { + match self { + MetricConfig::Counter(CounterConfig { field, .. }) => field, + MetricConfig::Histogram(HistogramConfig { field, .. }) => field, + MetricConfig::Gauge(GaugeConfig { field, .. }) => field, + MetricConfig::Set(SetConfig { field, .. }) => field, + MetricConfig::Summary(SummaryConfig { field, .. }) => field, + } + } +} + fn default_increment_by_value() -> bool { false } @@ -133,6 +145,9 @@ enum TransformError { FieldNotFound { field: String, }, + FieldNull { + field: String, + }, TemplateParseError(TemplateParseError), TemplateRenderingError(TemplateRenderingError), ParseFloatError { @@ -180,21 +195,6 @@ fn render_tags( }) } -fn parse_field(log: &LogEvent, field: &str) -> Result { - let value = log - .get(field) - .ok_or_else(|| TransformError::FieldNotFound { - field: field.to_string(), - })?; - value - .to_string_lossy() - .parse() - .map_err(|error| TransformError::ParseFloatError { - field: field.to_string(), - error, - }) -} - fn to_metric(config: &MetricConfig, event: &Event) -> Result { let log = event.as_log(); @@ -204,13 +204,20 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result Err(TransformError::FieldNotFound { + field: field.to_string(), + }), + Some(Value::Null) => Err(TransformError::FieldNull { + field: field.to_string(), + }), + Some(value) => Ok(value), + }?; + match config { MetricConfig::Counter(counter) => { - let value = log - .get(&counter.field) - .ok_or_else(|| TransformError::FieldNotFound { - field: counter.field.clone(), - })?; let value = if counter.increment_by_value { value.to_string_lossy().parse().map_err(|error| { TransformError::ParseFloatError { @@ -243,7 +250,12 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result { - let value = parse_field(&log, &hist.field)?; + let value = value.to_string_lossy().parse().map_err(|error| { + TransformError::ParseFloatError { + field: field.to_string(), + error, + } + })?; let name = hist.name.as_ref().unwrap_or(&hist.field); let name = render_template(&name, &event)?; @@ -269,7 +281,12 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result { - let value = parse_field(&log, &summary.field)?; + let value = value.to_string_lossy().parse().map_err(|error| { + TransformError::ParseFloatError { + field: field.to_string(), + error, + } + })?; let name = summary.name.as_ref().unwrap_or(&summary.field); let name = render_template(&name, &event)?; @@ -295,7 +312,12 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result { - let value = parse_field(&log, &gauge.field)?; + let value = value.to_string_lossy().parse().map_err(|error| { + TransformError::ParseFloatError { + field: field.to_string(), + error, + } + })?; let name = gauge.name.as_ref().unwrap_or(&gauge.field); let name = render_template(&name, &event)?; @@ -318,11 +340,6 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result { - let value = log - .get(&set.field) - .ok_or_else(|| TransformError::FieldNotFound { - field: set.field.clone(), - })?; let value = value.to_string_lossy(); let name = set.name.as_ref().unwrap_or(&set.field); @@ -357,6 +374,9 @@ impl FunctionTransform for LogToMetric { Ok(metric) => { output.push(Event::Metric(metric)); } + Err(TransformError::FieldNull { field }) => emit!(LogToMetricFieldNull { + field: field.as_ref() + }), Err(TransformError::FieldNotFound { field }) => emit!(LogToMetricFieldNotFound { field: field.as_ref() }), @@ -404,7 +424,7 @@ mod tests { Utc.ymd(2018, 11, 14).and_hms_nano(8, 9, 10, 11) } - fn create_event(key: &str, value: &str) -> Event { + fn create_event(key: &str, value: impl Into + std::fmt::Debug) -> Event { let mut log = Event::from("i am a log"); log.as_mut_log().insert(key, value); log.as_mut_log().insert(log_schema().timestamp_key(), ts()); @@ -618,6 +638,23 @@ mod tests { assert_eq!(transform.transform_one(event), None); } + #[test] + fn null_field() { + let config = parse_config( + r#" + [[metrics]] + type = "counter" + field = "status" + name = "status_total" + "#, + ); + + let event = create_event("status", Value::Null); + let mut transform = LogToMetric::new(config); + + assert_eq!(transform.transform_one(event), None); + } + #[test] fn multiple_metrics() { let config = parse_config(