Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: CEL filtering expression #658

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+222 −8
Draft

fix: CEL filtering expression #658

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions api/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,6 @@
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>

<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>${antlr4-maven-plugin.version}</version>
</dependency>

<dependency>
<groupId>org.opendatadiscovery</groupId>
<artifactId>oddrn-generator-java</artifactId>
Expand Down
104 changes: 104 additions & 0 deletions api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package io.kafbat.ui.emitter;

import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableSet;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelOptions;
import dev.cel.common.CelValidationException;
import dev.cel.common.CelValidationResult;
import dev.cel.common.types.CelType;
import dev.cel.common.types.CelTypeProvider;
import dev.cel.common.types.SimpleType;
import dev.cel.common.types.StructType;
import dev.cel.compiler.CelCompiler;
import dev.cel.compiler.CelCompilerFactory;
import dev.cel.extensions.CelExtensions;
import dev.cel.parser.CelStandardMacro;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import static org.assertj.core.api.Assertions.assertThat;

class CelTypedValidationTest {

private final StructType recordContentType = StructType.create(
"MessageContentDTO",
ImmutableSet.of("lastname", "firstname", "age"),
fieldName -> switch (fieldName) {
case "lastname", "firstname" ->
Optional.of(SimpleType.STRING);
case "age" ->
Optional.of(SimpleType.INT);
default ->
Optional.empty();
}
);

private final StructType recordType = StructType.create(
"MessageDTO",
ImmutableSet.of("value"),
fieldName -> Optional.of(recordContentType)
);

private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder()
.setOptions(CelOptions.DEFAULT)
.setStandardMacros(CelStandardMacro.STANDARD_MACROS)
.addLibraries(CelExtensions.strings(), CelExtensions.encoders())
.addVar("record", recordType)
.setResultType(SimpleType.BOOL)
.setTypeProvider(new CelTypeProvider() {
@Override
public ImmutableCollection<CelType> types() {
return ImmutableSet.of(recordType);
}

@Override
public Optional<CelType> findType(String typeName) {
if ("MessageDTO".equals(typeName)) {
return Optional.of(recordType);
}
if ("MessageContentDTO".equals(typeName)) {
return Optional.of(recordContentType);
}
return Optional.empty();
}
})
.build();

private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build();

@ParameterizedTest
@CsvSource({
"record.value.age == 24, true",
"record.value.age >= 24, true",
"record.value.age > 24, true",
"has(record.value.age), true",
"has(record.value.ages), false",
"record.value.age < 50, true",
"record.value.age < 24, true"
})
void typedExpressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException {
Map<String, Integer> objectKeys = new HashMap<>();
objectKeys.put("age", 24);
Map<String, Object> valueKeys = new HashMap<>();
valueKeys.put("value", objectKeys);
Map<String, Map<String, Object>> recordKeys = new HashMap<>();
recordKeys.put("record", valueKeys);

CelValidationResult celValidationResult = celCompiler.compile(expression);
CelAbstractSyntaxTree ast = celValidationResult.getAst();
CelRuntime.Program program = celRuntime.createProgram(ast);
var programResult = program.eval(recordKeys);
assertThat(programResult)
.describedAs("The result of the assertion was incorrect")
.isNotNull()
.isEqualTo(expected)
;
}
}
98 changes: 98 additions & 0 deletions api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package io.kafbat.ui.emitter;

import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableSet;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelOptions;
import dev.cel.common.CelValidationException;
import dev.cel.common.CelValidationResult;
import dev.cel.common.types.CelType;
import dev.cel.common.types.CelTypeProvider;
import dev.cel.common.types.SimpleType;
import dev.cel.common.types.StructType;
import dev.cel.compiler.CelCompiler;
import dev.cel.compiler.CelCompilerFactory;
import dev.cel.extensions.CelExtensions;
import dev.cel.parser.CelStandardMacro;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import static org.assertj.core.api.Assertions.assertThat;

class CelValidationTest {
private final Map<String, CelType> fields = Map.of(
"value", SimpleType.DYN
);

private final ImmutableSet<String> names = ImmutableSet
.<String>builder()
.addAll(fields.keySet())
.build();

private final StructType recordType = StructType.create(
"MessageDTO",
names,
fieldName -> Optional.ofNullable(fields.get(fieldName))
);

private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder()
.setOptions(CelOptions.DEFAULT)
.setStandardMacros(CelStandardMacro.STANDARD_MACROS)
.addLibraries(CelExtensions.strings(), CelExtensions.encoders())
.addVar("record", recordType)
.setResultType(SimpleType.BOOL)
.setTypeProvider(new CelTypeProvider() {
@Override
public ImmutableCollection<CelType> types() {
return ImmutableSet.of(recordType);
}

@Override
public Optional<CelType> findType(String typeName) {
return "MessageDTO".equals(typeName) ? Optional.of(recordType) : Optional.empty();
}
})
.build();

private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build();

@ParameterizedTest
@CsvSource({
"record.value.lastname == null, true",
"record.value.firstname == 'Paul', true",
"size(record.value.firstname) == 4, true",
"record.value.age == 24, true",
"record.value.age >= 24, true",
"record.value.age > 24, true",
"has(record.value.age), true",
"has(record.value.ages), false",
"record.value.age < 50, true",
"record.value.age < 24, true"
})
void expressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException {
Map<String, Object> objectKeys = new HashMap<>();
objectKeys.put("lastname", null);
objectKeys.put("firstname", "Paul");
objectKeys.put("age", 24);
Map<String, Object> valueKeys = new HashMap<>();
valueKeys.put("value", objectKeys);
Map<String, Map<String, Object>> recordKeys = new HashMap<>();
recordKeys.put("record", valueKeys);

CelValidationResult celValidationResult = celCompiler.compile(expression);
CelAbstractSyntaxTree ast = celValidationResult.getAst();
CelRuntime.Program program = celRuntime.createProgram(ast);
var programResult = program.eval(recordKeys);
assertThat(programResult)
.describedAs("The result of the assertion was incorrect")
.isNotNull()
.isEqualTo(expected)
;
}
}
18 changes: 18 additions & 0 deletions api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,24 @@ void canCheckValueAsJsonObjectIfItCanBeParsedToJson() {
assertFalse(f.test(msg().content("{ \"name\" : { \"second\" : \"user2\" } }")));
}

@Test
void canFilterNullValueInJsonObjectIfItCanBeParsedToJson() {
var filter1 = celScriptFilter("record.value.age == 24");
assertTrue(filter1.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Age filter KO");
var filter2 = celScriptFilter("record.value.name == 'Paul'");
assertTrue(filter2.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Name filter KO");
var filter3 = celScriptFilter("record.value.name == ''");
assertTrue(filter3.test(msg().content("{\"name\": \"\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Empty Name filter KO");
var filter4 = celScriptFilter("record.value.name == null");
assertFalse(filter4.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name in filter KO");
var filter5 = celScriptFilter("record.value.address.city == \"Lille\"");
assertTrue(filter5.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name filter KO");
var filter6 = celScriptFilter("record.value.address.city == \"\"");
assertTrue(filter6.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"\"}}")), "Null Name filter KO");
var filter7 = celScriptFilter("record.value.address.city == null");
assertTrue(filter7.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": null}}")), "Null Name filter KO");
}

@Test
void valueSetToContentStringIfCantBeParsedToJson() {
var f = celScriptFilter("record.value == \"not json\"");
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<git.revision>latest</git.revision>

<!-- Dependency versions -->
<antlr4-maven-plugin.version>4.12.0</antlr4-maven-plugin.version>
<antlr4-maven-plugin.version>4.13.2</antlr4-maven-plugin.version>
<apache.commons.version>2.12.0</apache.commons.version>
<assertj.version>3.25.3</assertj.version>
<avro.version>1.11.4</avro.version>
Expand All @@ -50,7 +50,7 @@
<odd-oddrn-generator.version>0.1.17</odd-oddrn-generator.version>
<odd-oddrn-client.version>0.1.39</odd-oddrn-client.version>
<org.json.version>20240303</org.json.version>
<dev.cel.version>0.3.0</dev.cel.version>
<dev.cel.version>0.7.0</dev.cel.version>
<guava.version>33.3.1-jre</guava.version>
<!-- Test dependency versions -->
<junit.version>5.11.2</junit.version>
Expand Down
Loading