Skip to content

filter_nest: add wildcard_exclude config #10112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
service:
log_level: debug
log_level: info

pipeline:
inputs:
@@ -20,9 +20,23 @@ pipeline:
"sub_key_1": "hello, world!",
"sub_key_2": "goodbye, world!"
}
}
},
"an_unknown_key": "hello, world!",
"another_unknown_key": "goodbye, world!"
}
filters:
- name: nest
match: logs
nest_under: nested_for_structured_metadata
wildcard: '*'
wildcard_exclude:
- message
- logger
- hostname
- level
- 'my_map_of_*'

outputs:
- name: loki
match: logs
@@ -31,6 +45,6 @@ pipeline:
label_keys: $level,$logger
labels: service_name=test
structured_metadata: $hostname
structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key']
structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key'],$nested_for_structured_metadata
line_format: key_value
drop_single_key: on
Original file line number Diff line number Diff line change
@@ -6,7 +6,8 @@ services:
# context: ../../
# dockerfile: dockerfiles/Dockerfile
depends_on:
- loki
loki:
condition: service_healthy
container_name: fluentbit
command: /fluent-bit/bin/fluent-bit -c /etc/fluent-bit_loki_out-structured_metadata_map.yaml
ports:
@@ -17,7 +18,8 @@ services:
- ./config/fluent-bit_loki_out-structured_metadata_map.yaml:/etc/fluent-bit_loki_out-structured_metadata_map.yaml

grafana:
image: grafana/grafana:11.4.0
image: grafana/grafana:11.5.2
attach: false
depends_on:
- loki
- fluentbit
@@ -28,11 +30,21 @@ services:
networks:
- loki-network
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_SECURITY_ADMIN=admin
- GF_SECURITY_ADMIN_PASSWORD=admin

loki:
image: grafana/loki:2.9.2
image: grafana/loki:3.4.2
attach: false
command: -config.file=/etc/loki/loki-config.yaml
healthcheck:
test: [ "CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:3100/ready" ]
interval: 5s
retries: 10
start_period: 10s
timeout: 10s
networks:
- loki-network
ports:
3 changes: 3 additions & 0 deletions docker_compose/node-exporter-dashboard/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -24,6 +24,9 @@ services:
networks:
- exporter-network
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_SECURITY_ADMIN=admin
- GF_SECURITY_ADMIN_PASSWORD=admin

prometheus:
65 changes: 64 additions & 1 deletion plugins/filter_nest/nest.c
Original file line number Diff line number Diff line change
@@ -41,6 +41,7 @@ static void teardown(struct filter_nest_ctx *ctx)
struct mk_list *head;

struct filter_nest_wildcard *wildcard;
struct filter_nest_wildcard *wildcard_excludes;

flb_free(ctx->prefix);
flb_free(ctx->key);
@@ -51,6 +52,12 @@ static void teardown(struct filter_nest_ctx *ctx)
mk_list_del(&wildcard->_head);
flb_free(wildcard);
}
mk_list_foreach_safe(head, tmp, &ctx->wildcard_excludes) {
wildcard_excludes = mk_list_entry(head, struct filter_nest_wildcard, _head);
flb_free(wildcard_excludes->key);
mk_list_del(&wildcard_excludes->_head);
flb_free(wildcard_excludes);
}

}

@@ -62,6 +69,7 @@ static int configure(struct filter_nest_ctx *ctx,
struct mk_list *head;
struct flb_kv *kv;
struct filter_nest_wildcard *wildcard;
struct filter_nest_wildcard *wildcard_exclude;

char *operation_nest = "nest";
char *operation_lift = "lift";
@@ -123,6 +131,35 @@ static int configure(struct filter_nest_ctx *ctx,
mk_list_add(&wildcard->_head, &ctx->wildcards);
ctx->wildcards_cnt++;

}
else if (strcasecmp(kv->key, "wildcard_exclude") == 0) {
wildcard_exclude = flb_malloc(sizeof(struct filter_nest_wildcard));
if (!wildcard_exclude) {
flb_plg_error(ctx->ins, "Unable to allocate memory for "
"wildcard_exclude");
flb_free(wildcard_exclude);
return -1;
}

wildcard_exclude->key = flb_strndup(kv->val, flb_sds_len(kv->val));
if (wildcard_exclude->key == NULL) {
flb_errno();
flb_free(wildcard_exclude);
return -1;
}
wildcard_exclude->key_len = flb_sds_len(kv->val);

if (wildcard_exclude->key[wildcard_exclude->key_len - 1] == '*') {
wildcard_exclude->key_is_dynamic = true;
wildcard_exclude->key_len--;
}
else {
wildcard_exclude->key_is_dynamic = false;
}

mk_list_add(&wildcard_exclude->_head, &ctx->wildcard_excludes);
ctx->wildcard_excludes_cnt++;

}
else if (strcasecmp(kv->key, "nest_under") == 0) {
ctx->key = flb_strdup(kv->val);
@@ -307,6 +344,7 @@ static inline bool is_kv_to_nest(msgpack_object_kv * kv,
struct mk_list *tmp;
struct mk_list *head;
struct filter_nest_wildcard *wildcard;
struct filter_nest_wildcard *wildcard_exclude;

if (obj->type == MSGPACK_OBJECT_BIN) {
key = obj->via.bin.ptr;
@@ -321,6 +359,24 @@ static inline bool is_kv_to_nest(msgpack_object_kv * kv,
return false;
}

mk_list_foreach_safe(head, tmp, &ctx->wildcard_excludes) {
wildcard_exclude = mk_list_entry(head, struct filter_nest_wildcard, _head);

if (wildcard_exclude->key_is_dynamic) {
/* This will negatively match "ABC123" with prefix "ABC*" */
if (strncmp(key, wildcard_exclude->key, wildcard_exclude->key_len) == 0) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

key is not a null terminated string and this comparison can potentially fail. Since klen is set, I recommend comparing first klen == wildcard_exclude->key_len and if it passes then do the strncmp()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, I copy and pasted this bloc of code from wildcards below, and adapted to wildcards_excludes.

As I read it, this part is for "dynamic" matches (where there is a wildcard), so we don't expect the lengths to match. For "non-dynamic" (which follows), it does do a length check already.

However, we could do a check here, something like klen >= wildcard_exclude->key_len, right?

This is to check that ABC123 is a viable match for wildcard ABC*, so we want to check that ABC123 is at least three chars long...

I will add some more unit tests to verify all this 🤞

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added the klen >= wildcard_exclude->key_len check, and some tests, and it all looks good

return false;
}
}
else {
/* This will negatively match "ABC" with prefix "ABC" */
if ((wildcard_exclude->key_len == klen) &&
(strncmp(key, wildcard_exclude->key, klen) == 0)
) {
return false;
}
}
}
mk_list_foreach_safe(head, tmp, &ctx->wildcards) {
wildcard = mk_list_entry(head, struct filter_nest_wildcard, _head);

@@ -615,9 +671,11 @@ static int cb_nest_init(struct flb_filter_instance *f_ins,
flb_errno();
return -1;
}
mk_list_init(&ctx->wildcards);
ctx->ins = f_ins;
mk_list_init(&ctx->wildcards);
ctx->wildcards_cnt = 0;
mk_list_init(&ctx->wildcard_excludes);
ctx->wildcard_excludes_cnt = 0;

if (configure(ctx, f_ins, config) < 0) {
flb_free(ctx);
@@ -737,6 +795,11 @@ static struct flb_config_map config_map[] = {
FLB_CONFIG_MAP_MULT, FLB_FALSE, 0,
"Nest records which field matches the wildcard"
},
{
FLB_CONFIG_MAP_STR, "Wildcard_exclude", NULL,
FLB_CONFIG_MAP_MULT, FLB_FALSE, 0,
"Nest records which field doesn't matches the wildcard"
},
{
FLB_CONFIG_MAP_STR, "Nest_under", NULL,
0, FLB_FALSE, 0,
2 changes: 2 additions & 0 deletions plugins/filter_nest/nest.h
Original file line number Diff line number Diff line change
@@ -38,6 +38,8 @@ struct filter_nest_ctx
// nest
struct mk_list wildcards;
int wildcards_cnt;
struct mk_list wildcard_excludes;
int wildcard_excludes_cnt;
bool remove_prefix;
// lift
bool add_prefix;
219 changes: 219 additions & 0 deletions tests/runtime/filter_nest.c
Original file line number Diff line number Diff line change
@@ -15,12 +15,18 @@ void flb_test_filter_nest_single(void);
void flb_test_filter_nest_multi_nest(void);
void flb_test_filter_nest_multi_lift(void);
void flb_test_filter_nest_add_prefix(void);
void flb_test_filter_nest_include_all(void);
void flb_test_filter_nest_exclude_static(void);
void flb_test_filter_nest_exclude_wildcard(void);
/* Test list */
TEST_LIST = {
{"single", flb_test_filter_nest_single },
{"multiple events are not dropped(nest)", flb_test_filter_nest_multi_nest},
{"multiple events are not dropped(lift)", flb_test_filter_nest_multi_lift},
{"add_prefix", flb_test_filter_nest_add_prefix},
{"include_all", flb_test_filter_nest_include_all},
{"exclude_static", flb_test_filter_nest_exclude_static},
{"exclude_wildcard", flb_test_filter_nest_exclude_wildcard},
{NULL, NULL}
};

@@ -372,3 +378,216 @@ void flb_test_filter_nest_add_prefix(void)
flb_stop(ctx);
flb_destroy(ctx);
}

void flb_test_filter_nest_include_all(void)
{
int ret;
int bytes;
char *p, *output, *expected;
flb_ctx_t *ctx;
int in_ffd;
int out_ffd;
int filter_ffd;
struct flb_lib_out_cb cb_data;

ctx = flb_create();
flb_service_set(ctx,
"Flush", "1",
"Grace", "1",
"Log_Level", "debug",
NULL);

/* Input */
in_ffd = flb_input(ctx, (char *) "lib", NULL);
TEST_CHECK(in_ffd >= 0);
flb_input_set(ctx, in_ffd, "tag", "test", NULL);

/* Prepare output callback context*/
cb_data.cb = callback_test;
cb_data.data = NULL;

/* Filter */
filter_ffd = flb_filter(ctx, (char *) "nest", NULL);
TEST_CHECK(filter_ffd >= 0);

ret = flb_filter_set(ctx, filter_ffd,
"Match", "*",
"Operation", "nest",
"Nest_under", "nested_key",
"Wildcard", "*",
NULL);

TEST_CHECK(ret == 0);


/* Output */
out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data);
TEST_CHECK(out_ffd >= 0);
flb_output_set(ctx, out_ffd,
"match", "test",
"format", "json",
NULL);

ret = flb_start(ctx);
TEST_CHECK(ret == 0);

p = "[1448403340, {\"to_nest\":\"This is some data to nest\", \"extra\":\"Some more data to nest\", \"EXCLUDED_extra_1\":\"Some more data to be excluded\", \"EXCLUDED_extra_2\":\"Some more data to be excluded\", \"EXCLUDED_extra_3\":\"Some more data to be excluded\"}]";
bytes = flb_lib_push(ctx, in_ffd, p, strlen(p));
TEST_CHECK(bytes == strlen(p));

flb_time_msleep(1500); /* waiting flush */
output = get_output();

TEST_CHECK_(output != NULL, "Expected output to not be NULL");

if (output != NULL) {
expected = "{\"nested_key\":{\"to_nest\":\"This is some data to nest\",\"extra\":\"Some more data to nest\",\"EXCLUDED_extra_1\":\"Some more data to be excluded\",\"EXCLUDED_extra_2\":\"Some more data to be excluded\",\"EXCLUDED_extra_3\":\"Some more data to be excluded\"}}]";
TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output);
free(output);
}
flb_stop(ctx);
flb_destroy(ctx);
}

void flb_test_filter_nest_exclude_static(void)
{
int ret;
int bytes;
char *p, *output, *expected;
flb_ctx_t *ctx;
int in_ffd;
int out_ffd;
int filter_ffd;
struct flb_lib_out_cb cb_data;

ctx = flb_create();
flb_service_set(ctx,
"Flush", "1",
"Grace", "1",
"Log_Level", "debug",
NULL);

/* Input */
in_ffd = flb_input(ctx, (char *) "lib", NULL);
TEST_CHECK(in_ffd >= 0);
flb_input_set(ctx, in_ffd, "tag", "test", NULL);

/* Prepare output callback context*/
cb_data.cb = callback_test;
cb_data.data = NULL;

/* Filter */
filter_ffd = flb_filter(ctx, (char *) "nest", NULL);
TEST_CHECK(filter_ffd >= 0);

ret = flb_filter_set(ctx, filter_ffd,
"Match", "*",
"Operation", "nest",
"Nest_under", "nested_key",
"Wildcard", "*",
"Wildcard_exclude", "EXCLUDED_extra_1",
"Wildcard_exclude", "EXCLUDED_extra_2",
NULL);

TEST_CHECK(ret == 0);


/* Output */
out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data);
TEST_CHECK(out_ffd >= 0);
flb_output_set(ctx, out_ffd,
"match", "test",
"format", "json",
NULL);

ret = flb_start(ctx);
TEST_CHECK(ret == 0);

p = "[1448403340, {\"to_nest\":\"This is some data to nest\", \"extra\":\"Some more data to nest\", \"EXCLUDED_extra_1\":\"Some more data to be excluded\", \"EXCLUDED_extra_2\":\"Some more data to be excluded\", \"EXCLUDED_NOT\":\"Some more data to be excluded\"}]";
bytes = flb_lib_push(ctx, in_ffd, p, strlen(p));
TEST_CHECK(bytes == strlen(p));

flb_time_msleep(1500); /* waiting flush */
output = get_output();

TEST_CHECK_(output != NULL, "Expected output to not be NULL");

if (output != NULL) {
expected = "\"nested_key\":{\"to_nest\":\"This is some data to nest\",\"extra\":\"Some more data to nest\",\"EXCLUDED_NOT\":\"Some more data to be excluded\"}}";
TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output);
free(output);
}
flb_stop(ctx);
flb_destroy(ctx);
}

void flb_test_filter_nest_exclude_wildcard(void)
{
int ret;
int bytes;
char *p, *output, *expected;
flb_ctx_t *ctx;
int in_ffd;
int out_ffd;
int filter_ffd;
struct flb_lib_out_cb cb_data;

ctx = flb_create();
flb_service_set(ctx,
"Flush", "1",
"Grace", "1",
"Log_Level", "debug",
NULL);

/* Input */
in_ffd = flb_input(ctx, (char *) "lib", NULL);
TEST_CHECK(in_ffd >= 0);
flb_input_set(ctx, in_ffd, "tag", "test", NULL);

/* Prepare output callback context*/
cb_data.cb = callback_test;
cb_data.data = NULL;

/* Filter */
filter_ffd = flb_filter(ctx, (char *) "nest", NULL);
TEST_CHECK(filter_ffd >= 0);

ret = flb_filter_set(ctx, filter_ffd,
"Match", "*",
"Operation", "nest",
"Nest_under", "nested_key",
"Wildcard", "*",
"Wildcard_exclude", "EXCLUDED_*",
NULL);

TEST_CHECK(ret == 0);


/* Output */
out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data);
TEST_CHECK(out_ffd >= 0);
flb_output_set(ctx, out_ffd,
"match", "test",
"format", "json",
NULL);

ret = flb_start(ctx);
TEST_CHECK(ret == 0);

p = "[1448403340, {\"to_nest\":\"This is some data to nest\", \"extra\":\"Some more data to nest\", \"EXCLUDED_extra_1\":\"Some more data to be excluded\", \"EXCLUDED_extra_2\":\"Some more data to be excluded\", \"EXCLUDED_extra_3\":\"Some more data to be excluded\"}]";
bytes = flb_lib_push(ctx, in_ffd, p, strlen(p));
TEST_CHECK(bytes == strlen(p));

flb_time_msleep(1500); /* waiting flush */
output = get_output();

TEST_CHECK_(output != NULL, "Expected output to not be NULL");

if (output != NULL) {
expected = "\"nested_key\":{\"to_nest\":\"This is some data to nest\",\"extra\":\"Some more data to nest\"}}]";
TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output);
free(output);
}
flb_stop(ctx);
flb_destroy(ctx);
}