From fee8b9e1cf525f8eec433d6d2e7321656712d3fa Mon Sep 17 00:00:00 2001 From: Douglas SIX Date: Mon, 11 Nov 2024 20:13:49 +0100 Subject: [PATCH] fix: CEL filtering expression Trying multiple things here: - bump CEL version - add test in `MessageFiltersTest` on different fields of different types - add `CelValidationTest' and 'CelTypedValidationTest' to try to understand how CEL works --- api/pom.xml | 6 - .../ui/emitter/CelTypedValidationTest.java | 104 ++++++++++++++++++ .../kafbat/ui/emitter/CelValidationTest.java | 98 +++++++++++++++++ .../kafbat/ui/emitter/MessageFiltersTest.java | 18 +++ pom.xml | 4 +- 5 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java create mode 100644 api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java diff --git a/api/pom.xml b/api/pom.xml index 8452167ce..4aeb80bc7 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -215,12 +215,6 @@ spring-boot-starter-actuator - - org.antlr - antlr4-runtime - ${antlr4-maven-plugin.version} - - org.opendatadiscovery oddrn-generator-java diff --git a/api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java b/api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java new file mode 100644 index 000000000..c82cc1a1b --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java @@ -0,0 +1,104 @@ +package io.kafbat.ui.emitter; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableSet; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelOptions; +import dev.cel.common.CelValidationException; +import dev.cel.common.CelValidationResult; +import dev.cel.common.types.CelType; +import dev.cel.common.types.CelTypeProvider; +import dev.cel.common.types.SimpleType; +import dev.cel.common.types.StructType; +import dev.cel.compiler.CelCompiler; +import dev.cel.compiler.CelCompilerFactory; +import dev.cel.extensions.CelExtensions; +import dev.cel.parser.CelStandardMacro; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntimeFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class CelTypedValidationTest { + + private final StructType recordContentType = StructType.create( + "MessageContentDTO", + ImmutableSet.of("lastname", "firstname", "age"), + fieldName -> switch (fieldName) { + case "lastname", "firstname" -> + Optional.of(SimpleType.STRING); + case "age" -> + Optional.of(SimpleType.INT); + default -> + Optional.empty(); + } + ); + + private final StructType recordType = StructType.create( + "MessageDTO", + ImmutableSet.of("value"), + fieldName -> Optional.of(recordContentType) + ); + + private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder() + .setOptions(CelOptions.DEFAULT) + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .addLibraries(CelExtensions.strings(), CelExtensions.encoders()) + .addVar("record", recordType) + .setResultType(SimpleType.BOOL) + .setTypeProvider(new CelTypeProvider() { + @Override + public ImmutableCollection types() { + return ImmutableSet.of(recordType); + } + + @Override + public Optional findType(String typeName) { + if ("MessageDTO".equals(typeName)) { + return Optional.of(recordType); + } + if ("MessageContentDTO".equals(typeName)) { + return Optional.of(recordContentType); + } + return Optional.empty(); + } + }) + .build(); + + private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build(); + + @ParameterizedTest + @CsvSource({ + "record.value.age == 24, true", + "record.value.age >= 24, true", + "record.value.age > 24, true", + "has(record.value.age), true", + "has(record.value.ages), false", + "record.value.age < 50, true", + "record.value.age < 24, true" + }) + void typedExpressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException { + Map objectKeys = new HashMap<>(); + objectKeys.put("age", 24); + Map valueKeys = new HashMap<>(); + valueKeys.put("value", objectKeys); + Map> recordKeys = new HashMap<>(); + recordKeys.put("record", valueKeys); + + CelValidationResult celValidationResult = celCompiler.compile(expression); + CelAbstractSyntaxTree ast = celValidationResult.getAst(); + CelRuntime.Program program = celRuntime.createProgram(ast); + var programResult = program.eval(recordKeys); + assertThat(programResult) + .describedAs("The result of the assertion was incorrect") + .isNotNull() + .isEqualTo(expected) + ; + } +} diff --git a/api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java b/api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java new file mode 100644 index 000000000..4df8e698f --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java @@ -0,0 +1,98 @@ +package io.kafbat.ui.emitter; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableSet; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelOptions; +import dev.cel.common.CelValidationException; +import dev.cel.common.CelValidationResult; +import dev.cel.common.types.CelType; +import dev.cel.common.types.CelTypeProvider; +import dev.cel.common.types.SimpleType; +import dev.cel.common.types.StructType; +import dev.cel.compiler.CelCompiler; +import dev.cel.compiler.CelCompilerFactory; +import dev.cel.extensions.CelExtensions; +import dev.cel.parser.CelStandardMacro; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntimeFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class CelValidationTest { + private final Map fields = Map.of( + "value", SimpleType.DYN + ); + + private final ImmutableSet names = ImmutableSet + .builder() + .addAll(fields.keySet()) + .build(); + + private final StructType recordType = StructType.create( + "MessageDTO", + names, + fieldName -> Optional.ofNullable(fields.get(fieldName)) + ); + + private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder() + .setOptions(CelOptions.DEFAULT) + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .addLibraries(CelExtensions.strings(), CelExtensions.encoders()) + .addVar("record", recordType) + .setResultType(SimpleType.BOOL) + .setTypeProvider(new CelTypeProvider() { + @Override + public ImmutableCollection types() { + return ImmutableSet.of(recordType); + } + + @Override + public Optional findType(String typeName) { + return "MessageDTO".equals(typeName) ? Optional.of(recordType) : Optional.empty(); + } + }) + .build(); + + private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build(); + + @ParameterizedTest + @CsvSource({ + "record.value.lastname == null, true", + "record.value.firstname == 'Paul', true", + "size(record.value.firstname) == 4, true", + "record.value.age == 24, true", + "record.value.age >= 24, true", + "record.value.age > 24, true", + "has(record.value.age), true", + "has(record.value.ages), false", + "record.value.age < 50, true", + "record.value.age < 24, true" + }) + void expressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException { + Map objectKeys = new HashMap<>(); + objectKeys.put("lastname", null); + objectKeys.put("firstname", "Paul"); + objectKeys.put("age", 24); + Map valueKeys = new HashMap<>(); + valueKeys.put("value", objectKeys); + Map> recordKeys = new HashMap<>(); + recordKeys.put("record", valueKeys); + + CelValidationResult celValidationResult = celCompiler.compile(expression); + CelAbstractSyntaxTree ast = celValidationResult.getAst(); + CelRuntime.Program program = celRuntime.createProgram(ast); + var programResult = program.eval(recordKeys); + assertThat(programResult) + .describedAs("The result of the assertion was incorrect") + .isNotNull() + .isEqualTo(expected) + ; + } +} diff --git a/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java b/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java index 617cfb0c1..c97ba71e6 100644 --- a/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java +++ b/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java @@ -149,6 +149,24 @@ void canCheckValueAsJsonObjectIfItCanBeParsedToJson() { assertFalse(f.test(msg().content("{ \"name\" : { \"second\" : \"user2\" } }"))); } + @Test + void canFilterNullValueInJsonObjectIfItCanBeParsedToJson() { + var filter1 = celScriptFilter("record.value.age == 24"); + assertTrue(filter1.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Age filter KO"); + var filter2 = celScriptFilter("record.value.name == 'Paul'"); + assertTrue(filter2.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Name filter KO"); + var filter3 = celScriptFilter("record.value.name == ''"); + assertTrue(filter3.test(msg().content("{\"name\": \"\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Empty Name filter KO"); + var filter4 = celScriptFilter("record.value.name == null"); + assertFalse(filter4.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name in filter KO"); + var filter5 = celScriptFilter("record.value.address.city == \"Lille\""); + assertTrue(filter5.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name filter KO"); + var filter6 = celScriptFilter("record.value.address.city == \"\""); + assertTrue(filter6.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"\"}}")), "Null Name filter KO"); + var filter7 = celScriptFilter("record.value.address.city == null"); + assertTrue(filter7.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": null}}")), "Null Name filter KO"); + } + @Test void valueSetToContentStringIfCantBeParsedToJson() { var f = celScriptFilter("record.value == \"not json\""); diff --git a/pom.xml b/pom.xml index e38658fbb..78c7707cb 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ latest - 4.12.0 + 4.13.2 2.12.0 3.25.3 1.11.4 @@ -50,7 +50,7 @@ 0.1.17 0.1.39 20240303 - 0.3.0 + 0.7.0 33.3.1-jre 5.11.2