From 82632248de93926d1d065fa63b6efb3a9b8394a7 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Fri, 18 Aug 2023 22:26:42 +0100 Subject: [PATCH 1/3] feat(route transform): Add option to enable/disable unmatched output This commit adds a new boolean option `reroute_unmatched` to the `route` transform. It is inspired on the `reroute_dropped` option in the `remap` transform, allowing the user to control if they want or not the `._unmatched` output to be created and used. For backwards compatibility, this new option defaults to `true`. Users that are not interested in processing unmatched events can use this option to avoid the following warning on Vector startup, and also to remove the unnecessary `_unmatched` output in `vector top`: ``` WARN vector::config::loading: Transform "route._unmatched" has no consumers ``` Signed-off-by: Hugo Hromic --- src/transforms/route.rs | 36 +++++++++++---- .../components/transforms/base/route.cue | 45 +++++++++++++------ 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/transforms/route.rs b/src/transforms/route.rs index e410277914a8f..b64f5c9158b82 100644 --- a/src/transforms/route.rs +++ b/src/transforms/route.rs @@ -19,6 +19,7 @@ pub(crate) const UNMATCHED_ROUTE: &str = "_unmatched"; #[derive(Clone)] pub struct Route { conditions: Vec<(String, Condition)>, + reroute_unmatched: bool, } impl Route { @@ -28,7 +29,10 @@ impl Route { let condition = condition.build(&context.enrichment_tables)?; conditions.push((output_name.clone(), condition)); } - Ok(Self { conditions }) + Ok(Self { + conditions, + reroute_unmatched: config.reroute_unmatched, + }) } } @@ -47,7 +51,7 @@ impl SyncTransform for Route { check_failed += 1; } } - if check_failed == self.conditions.len() { + if self.reroute_unmatched && check_failed == self.conditions.len() { output.push(Some(UNMATCHED_ROUTE), event); } } @@ -61,11 +65,24 @@ impl SyncTransform for Route { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct RouteConfig { + /// Reroutes unmatched events to a named output instead of silently discarding them. + /// + /// Normally, if an event doesn't match any defined route, it is sent to the `._unmatched` + /// output for further processing. In some cases, you may want to simply discard unmatched events and not + /// process them any further. + /// + /// In these cases, `reroute_unmatched` can be set to `false` to disable the `._unmatched` + /// output and instead silently discard any unmatched events. + #[serde(default = "crate::serde::default_true")] + #[configurable(metadata(docs::human_name = "Reroute Unmatched Events"))] + reroute_unmatched: bool, + /// A table of route identifiers to logical conditions representing the filter of the route. /// /// Each route can then be referenced as an input by other components with the name - /// `.`. If an event doesn’t match any route, it is sent to the - /// `._unmatched` output. + /// `.`. If an event doesn’t match any route, and if `reroute_unmatched` + /// is set to `true` (the default), it is sent to the `._unmatched` output. + /// Otherwise, the unmatched event is instead silently discarded. /// /// Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used /// as a route name. @@ -76,6 +93,7 @@ pub struct RouteConfig { impl GenerateConfig for RouteConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { + reroute_unmatched: true, route: IndexMap::new(), }) .unwrap() @@ -118,10 +136,12 @@ impl TransformConfig for RouteConfig { .with_port(output_name) }) .collect(); - result.push( - TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions)) - .with_port(UNMATCHED_ROUTE), - ); + if self.reroute_unmatched { + result.push( + TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions)) + .with_port(UNMATCHED_ROUTE), + ); + } result } diff --git a/website/cue/reference/components/transforms/base/route.cue b/website/cue/reference/components/transforms/base/route.cue index 873757ec82411..ad6e02580c813 100644 --- a/website/cue/reference/components/transforms/base/route.cue +++ b/website/cue/reference/components/transforms/base/route.cue @@ -1,20 +1,37 @@ package metadata -base: components: transforms: route: configuration: route: { - description: """ - A table of route identifiers to logical conditions representing the filter of the route. +base: components: transforms: route: configuration: { + reroute_unmatched: { + description: """ + Reroutes unmatched events to a named output instead of silently discarding them. - Each route can then be referenced as an input by other components with the name - `.`. If an event doesn’t match any route, it is sent to the - `._unmatched` output. + Normally, if an event doesn't match any defined route, it is sent to the `._unmatched` + output for further processing. In some cases, you may want to simply discard unmatched events and not + process them any further. - Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used - as a route name. - """ - required: false - type: object: options: "*": { - description: "An individual route." - required: true - type: condition: {} + In these cases, `reroute_unmatched` can be set to `false` to disable the `._unmatched` + output and instead silently discard any unmatched events. + """ + required: false + type: bool: default: true + } + route: { + description: """ + A table of route identifiers to logical conditions representing the filter of the route. + + Each route can then be referenced as an input by other components with the name + `.`. If an event doesn’t match any route, and if `reroute_unmatched` + is set to `true` (the default), it is sent to the `._unmatched` output. + Otherwise, the unmatched event is instead silently discarded. + + Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used + as a route name. + """ + required: false + type: object: options: "*": { + description: "An individual route." + required: true + type: condition: {} + } } } From 91cec1ad1e97a7b350fd4292a0e270a55a937c37 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Sat, 19 Aug 2023 00:03:31 +0100 Subject: [PATCH 2/3] Update existing `can_serialize_remap` test --- src/transforms/route.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transforms/route.rs b/src/transforms/route.rs index b64f5c9158b82..e6a3025a574ed 100644 --- a/src/transforms/route.rs +++ b/src/transforms/route.rs @@ -182,7 +182,7 @@ mod test { assert_eq!( serde_json::to_string(&config).unwrap(), - r#"{"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"# + r#"{"reroute_unmatched":true,"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"# ); } From ebf0baf25f41c0e1338d41a259721fe338496021 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Sat, 19 Aug 2023 00:30:46 +0100 Subject: [PATCH 3/3] Add test for the new `reroute_unmatched` option --- src/transforms/route.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/transforms/route.rs b/src/transforms/route.rs index e6a3025a574ed..721d92a9789ac 100644 --- a/src/transforms/route.rs +++ b/src/transforms/route.rs @@ -313,6 +313,45 @@ mod test { } } + #[test] + fn route_no_unmatched_output() { + let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE]; + let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap(); + let config = toml::from_str::( + r#" + reroute_unmatched = false + + route.first.type = "vrl" + route.first.source = '.message == "hello world"' + + route.second.type = "vrl" + route.second.source = '.second == "second"' + + route.third.type = "vrl" + route.third.source = '.third == "third"' + "#, + ) + .unwrap(); + + let mut transform = Route::new(&config, &Default::default()).unwrap(); + let mut outputs = TransformOutputsBuf::new_with_capacity( + output_names + .iter() + .map(|output_name| { + TransformOutput::new(DataType::all(), HashMap::new()) + .with_port(output_name.to_owned()) + }) + .collect(), + 1, + ); + + transform.transform(event.clone(), &mut outputs); + for output_name in output_names { + let events: Vec<_> = outputs.drain_named(output_name).collect(); + assert_eq!(events.len(), 0); + } + } + #[tokio::test] async fn route_metrics_with_output_tag() { init_test();