Skip to content

Commit 429e8a9

Browse files
duhizjameduhizjame
committed
protocol not compliant
1 parent b9ae779 commit 429e8a9

File tree

6 files changed

+137
-31
lines changed

6 files changed

+137
-31
lines changed

‎server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,56 @@ public void getTableVersionBadTimestamp() {
292292
.statusCode(502);
293293
}
294294

295+
@DisabledOnOs(OS.WINDOWS)
296+
@Test
297+
public void queryTableCurrentVersionWithPredicates() throws IOException {
298+
var responseBodyLines = given()
299+
.when()
300+
.filter(deltaFilter)
301+
.body("{\"jsonPredicateHints\": {\"op\":\"and\",\"children\":\"[" +
302+
" {op:not,children:[" +
303+
" {op:isNull\",\"children\":[\n" +
304+
" {\"op\":\"column\",\"name\":\"birthday\",\"valueType\":\"date\"}]}]},\n" +
305+
" {\"op\":\"equal\",\"children\":[\n" +
306+
" {\"op\":\"column\",\"name\":\"birthday\",\"valueType\":\"date\"},\n" +
307+
" {\"op\":\"literal\",\"value\":\"2020-01-01\",\"valueType\":\"date\"}]}\n" +
308+
"]\"}}")
309+
.header(new Header("Content-Type", "application/json"))
310+
.post(
311+
"delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query",
312+
"name",
313+
"default",
314+
"table1")
315+
.then()
316+
.statusCode(200)
317+
.extract()
318+
.body()
319+
.asString()
320+
.split("\n");
321+
322+
assertEquals(
323+
deltaTable1Protocol,
324+
objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class));
325+
assertEquals(
326+
deltaTable1Metadata,
327+
objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class));
328+
var files = Arrays.stream(responseBodyLines)
329+
.skip(2)
330+
.map(line -> {
331+
try {
332+
return objectMapper
333+
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
334+
.reader()
335+
.readValue(line, FileObjectWithoutPresignedUrl.class);
336+
} catch (IOException e) {
337+
throw new RuntimeException(e);
338+
}
339+
})
340+
.collect(Collectors.toSet());
341+
assertEquals(7, responseBodyLines.length);
342+
assertEquals(deltaTable1FilesWithoutPresignedUrl, files); // TOD
343+
}
344+
295345
@DisabledOnOs(OS.WINDOWS)
296346
@Test
297347
public void queryTableCurrentVersion() throws IOException {

‎server/core/src/main/java/io/whitefox/core/PredicateUtils.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,12 @@
1414
import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
1515
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
1616
import org.apache.commons.lang3.tuple.Pair;
17+
import org.apache.log4j.Logger;
1718

1819
public class PredicateUtils {
1920

21+
private static final Logger logger = Logger.getLogger(PredicateUtils.class);
22+
2023
private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance();
2124

2225
public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsingException {
@@ -27,6 +30,34 @@ public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsin
2730
}
2831
}
2932

33+
34+
public static boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) {
35+
try {
36+
var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate);
37+
return parsedPredicate.evalExpectBoolean(ctx);
38+
} catch (PredicateException e) {
39+
logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
40+
logger.info("File: " + f.getPath()
41+
+ " will be used in processing due to failure in parsing or processing the predicate: "
42+
+ predicate);
43+
return true;
44+
}
45+
}
46+
47+
public static boolean evaluateSqlPredicate(
48+
String predicate, EvalContext ctx, AddFile f, Metadata metadata) {
49+
try {
50+
var parsedPredicate = PredicateUtils.parseSqlPredicate(predicate, ctx, metadata);
51+
return parsedPredicate.evalExpectBoolean(ctx);
52+
} catch (PredicateException e) {
53+
logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
54+
logger.info("File: " + f.getPath()
55+
+ " will be used in processing due to failure in parsing or processing the predicate: "
56+
+ predicate);
57+
return true;
58+
}
59+
}
60+
3061
public static BaseOp parseSqlPredicate(String predicate, EvalContext ctx, Metadata metadata)
3162
throws PredicateException {
3263
try {

‎server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java‎

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package io.whitefox.core.services;
22

3+
import static io.whitefox.core.PredicateUtils.evaluateJsonPredicate;
4+
import static io.whitefox.core.PredicateUtils.evaluateSqlPredicate;
5+
36
import io.delta.standalone.DeltaLog;
47
import io.delta.standalone.Snapshot;
58
import io.delta.standalone.actions.AddFile;
@@ -86,33 +89,6 @@ public Optional<Long> getTableVersion(Optional<String> startingTimestamp) {
8689
return getSnapshot(startingTimestamp).map(Snapshot::getVersion);
8790
}
8891

89-
private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) {
90-
try {
91-
var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate);
92-
return parsedPredicate.evalExpectBoolean(ctx);
93-
} catch (PredicateException e) {
94-
logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
95-
logger.info("File: " + f.getPath()
96-
+ " will be used in processing due to failure in parsing or processing the predicate: "
97-
+ predicate);
98-
return true;
99-
}
100-
}
101-
102-
private boolean evaluateSqlPredicate(
103-
String predicate, EvalContext ctx, AddFile f, Metadata metadata) {
104-
try {
105-
var parsedPredicate = PredicateUtils.parseSqlPredicate(predicate, ctx, metadata);
106-
return parsedPredicate.evalExpectBoolean(ctx);
107-
} catch (PredicateException e) {
108-
logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
109-
logger.info("File: " + f.getPath()
110-
+ " will be used in processing due to failure in parsing or processing the predicate: "
111-
+ predicate);
112-
return true;
113-
}
114-
}
115-
11692
public boolean filterFilesBasedOnSqlPredicates(
11793
List<String> predicates, AddFile f, Metadata metadata) {
11894
// if there are no predicates return all possible files

‎server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) {
111111

112112
// Validates that the specified value is in the correct format.
113113
// Throws an exception otherwise.
114-
public static void validateValue(String value, DataType valueType) {
114+
public static void validateValue(String value, DataType valueType)
115+
throws TypeValidationException {
115116
try {
116117
if (BooleanType.BOOLEAN.equals(valueType)) {
117118
Boolean.parseBoolean(value);
@@ -134,8 +135,7 @@ public static void validateValue(String value, DataType valueType) {
134135
throw new TypeNotSupportedException(valueType);
135136
}
136137
} catch (Exception e) {
137-
throw new IllegalArgumentException(
138-
"Error validating " + value + " for type " + valueType + ": " + e);
138+
throw new TypeValidationException(value, valueType);
139139
}
140140
}
141141
}

‎server/core/src/main/java/io/whitefox/core/types/predicates/TypeValidationException.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package io.whitefox.core.types.predicates;
2+
3+
import io.whitefox.core.types.DataType;
4+
5+
public class TypeValidationException extends PredicateException {
6+
7+
private final String value;
8+
private final DataType valueType;
9+
10+
public TypeValidationException(String value, DataType valueType) {
11+
this.value = value;
12+
this.valueType = valueType;
13+
}
14+
15+
@Override
16+
public String getMessage() {
17+
return "Error validating value: " + value + " for type " + valueType;
18+
}
19+
}

‎server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,21 @@
22

33
import static org.junit.jupiter.api.Assertions.assertThrows;
44

5+
import io.whitefox.core.Metadata;
56
import io.whitefox.core.PredicateUtils;
7+
import io.whitefox.core.TableSchema;
8+
import io.whitefox.core.types.DateType;
9+
import io.whitefox.core.types.StructField;
10+
import io.whitefox.core.types.StructType;
11+
import java.util.List;
12+
import java.util.Map;
13+
import java.util.Optional;
614
import org.junit.jupiter.api.Test;
715

816
public class PredicateParsingTest {
917

1018
@Test
11-
void testParsingOfEqual() throws PredicateException {
19+
void testParsingOfJsonEqual() throws PredicateException {
1220

1321
var predicate = "{\n" + " \"op\": \"equal\",\n"
1422
+ " \"children\": [\n"
@@ -22,6 +30,28 @@ void testParsingOfEqual() throws PredicateException {
2230
assert (((EqualOp) op).children.size() == 2);
2331
}
2432

33+
@Test
34+
void testParsingOfSqlEqual() throws PredicateException {
35+
var ctx = new EvalContext(Map.of("date", "date"), Map.of());
36+
var meta = new Metadata(
37+
"id",
38+
Optional.empty(),
39+
Optional.empty(),
40+
Metadata.Format.PARQUET,
41+
new TableSchema(
42+
new StructType(List.of(new StructField("date", DateType.DATE, true, Map.of())))),
43+
List.of("date", "age"),
44+
Map.of(),
45+
Optional.empty(),
46+
Optional.empty(),
47+
Optional.empty());
48+
var predicate = "date = '2021-09-09'";
49+
var op = PredicateUtils.parseSqlPredicate(predicate, ctx, meta);
50+
op.validate();
51+
assert (op instanceof EqualOp);
52+
assert (((EqualOp) op).children.size() == 2);
53+
}
54+
2555
@Test
2656
void testParsingOfNested() throws PredicateException {
2757
var predicate = "{\n" + " \"op\":\"and\",\n"

0 commit comments

Comments
 (0)