Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(route transform): Add option to enable/disable unmatched output #18309

Merged
merged 3 commits into from
Aug 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 68 additions & 9 deletions src/transforms/route.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ pub(crate) const UNMATCHED_ROUTE: &str = "_unmatched";
#[derive(Clone)]
pub struct Route {
conditions: Vec<(String, Condition)>,
reroute_unmatched: bool,
}

impl Route {
Expand All @@ -28,7 +29,10 @@ impl Route {
let condition = condition.build(&context.enrichment_tables)?;
conditions.push((output_name.clone(), condition));
}
Ok(Self { conditions })
Ok(Self {
conditions,
reroute_unmatched: config.reroute_unmatched,
})
}
}

Expand All @@ -47,7 +51,7 @@ impl SyncTransform for Route {
check_failed += 1;
}
}
if check_failed == self.conditions.len() {
if self.reroute_unmatched && check_failed == self.conditions.len() {
output.push(Some(UNMATCHED_ROUTE), event);
}
}
Expand All @@ -61,11 +65,24 @@ impl SyncTransform for Route {
#[derive(Clone, Debug)]
#[serde(deny_unknown_fields)]
pub struct RouteConfig {
/// Reroutes unmatched events to a named output instead of silently discarding them.
///
/// Normally, if an event doesn't match any defined route, it is sent to the `<transform_name>._unmatched`
/// output for further processing. In some cases, you may want to simply discard unmatched events and not
/// process them any further.
///
/// In these cases, `reroute_unmatched` can be set to `false` to disable the `<transform_name>._unmatched`
/// output and instead silently discard any unmatched events.
#[serde(default = "crate::serde::default_true")]
#[configurable(metadata(docs::human_name = "Reroute Unmatched Events"))]
reroute_unmatched: bool,

/// A table of route identifiers to logical conditions representing the filter of the route.
///
/// Each route can then be referenced as an input by other components with the name
/// `<transform_name>.<route_id>`. If an event doesn’t match any route, it is sent to the
/// `<transform_name>._unmatched` output.
/// `<transform_name>.<route_id>`. If an event doesn’t match any route, and if `reroute_unmatched`
/// is set to `true` (the default), it is sent to the `<transform_name>._unmatched` output.
/// Otherwise, the unmatched event is instead silently discarded.
///
/// Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used
/// as a route name.
Expand All @@ -76,6 +93,7 @@ pub struct RouteConfig {
impl GenerateConfig for RouteConfig {
fn generate_config() -> toml::Value {
toml::Value::try_from(Self {
reroute_unmatched: true,
route: IndexMap::new(),
})
.unwrap()
Expand Down Expand Up @@ -118,10 +136,12 @@ impl TransformConfig for RouteConfig {
.with_port(output_name)
})
.collect();
result.push(
TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions))
.with_port(UNMATCHED_ROUTE),
);
if self.reroute_unmatched {
result.push(
TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions))
.with_port(UNMATCHED_ROUTE),
);
}
result
}

Expand Down Expand Up @@ -162,7 +182,7 @@ mod test {

assert_eq!(
serde_json::to_string(&config).unwrap(),
r#"{"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"#
r#"{"reroute_unmatched":true,"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"#
);
}

Expand Down Expand Up @@ -293,6 +313,45 @@ mod test {
}
}

#[test]
fn route_no_unmatched_output() {
let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE];
let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap();
let config = toml::from_str::<RouteConfig>(
r#"
reroute_unmatched = false

route.first.type = "vrl"
route.first.source = '.message == "hello world"'

route.second.type = "vrl"
route.second.source = '.second == "second"'

route.third.type = "vrl"
route.third.source = '.third == "third"'
"#,
)
.unwrap();

let mut transform = Route::new(&config, &Default::default()).unwrap();
let mut outputs = TransformOutputsBuf::new_with_capacity(
output_names
.iter()
.map(|output_name| {
TransformOutput::new(DataType::all(), HashMap::new())
.with_port(output_name.to_owned())
})
.collect(),
1,
);

transform.transform(event.clone(), &mut outputs);
for output_name in output_names {
let events: Vec<_> = outputs.drain_named(output_name).collect();
assert_eq!(events.len(), 0);
}
}

#[tokio::test]
async fn route_metrics_with_output_tag() {
init_test();
Expand Down
45 changes: 31 additions & 14 deletions website/cue/reference/components/transforms/base/route.cue
Original file line number Diff line number Diff line change
@@ -1,20 +1,37 @@
package metadata

base: components: transforms: route: configuration: route: {
description: """
A table of route identifiers to logical conditions representing the filter of the route.
base: components: transforms: route: configuration: {
reroute_unmatched: {
description: """
Reroutes unmatched events to a named output instead of silently discarding them.

Each route can then be referenced as an input by other components with the name
`<transform_name>.<route_id>`. If an event doesn’t match any route, it is sent to the
`<transform_name>._unmatched` output.
Normally, if an event doesn't match any defined route, it is sent to the `<transform_name>._unmatched`
output for further processing. In some cases, you may want to simply discard unmatched events and not
process them any further.

Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used
as a route name.
"""
required: false
type: object: options: "*": {
description: "An individual route."
required: true
type: condition: {}
In these cases, `reroute_unmatched` can be set to `false` to disable the `<transform_name>._unmatched`
output and instead silently discard any unmatched events.
"""
required: false
type: bool: default: true
}
route: {
description: """
A table of route identifiers to logical conditions representing the filter of the route.

Each route can then be referenced as an input by other components with the name
`<transform_name>.<route_id>`. If an event doesn’t match any route, and if `reroute_unmatched`
is set to `true` (the default), it is sent to the `<transform_name>._unmatched` output.
Otherwise, the unmatched event is instead silently discarded.

Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used
as a route name.
"""
required: false
type: object: options: "*": {
description: "An individual route."
required: true
type: condition: {}
}
}
}