From 79a7718f9f33e9a0d3d98ce99d736a54e4c98701 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Sun, 7 Apr 2024 15:02:37 +0800 Subject: [PATCH] [fix](iceberg) fix iceberg predicate conversion bug (#33283) Followup #32923 Some cases are not covered in #32923 --- .../datasource/iceberg/IcebergUtils.java | 73 ++- .../iceberg/source/IcebergScanNode.java | 18 +- .../iceberg/IcebergPredicateTest.java | 256 ++++++++ .../iceberg/TestIcebergPredict.java | 135 ---- .../test_iceberg_predicate_conversion.out | 611 ++++++++++++++++++ .../test_iceberg_predicate_conversion.groovy | 79 +++ 6 files changed, 1028 insertions(+), 144 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java create mode 100644 regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out create mode 100644 regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 76b5bfb5105c4f..08c4be4ceace20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -48,8 +48,11 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.expressions.And; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Not; +import org.apache.iceberg.expressions.Or; import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; @@ -107,6 +110,10 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) { Expression right = convertToIcebergExpr(compoundPredicate.getChild(1), schema); if (left != null && right != null) { expression = Expressions.and(left, right); + } else if (left != null) { + return left; + } else if (right != null) { + return right; } break; } @@ -209,6 +216,9 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) { } LiteralExpr literalExpr = (LiteralExpr) inExpr.getChild(i); Object value = extractDorisLiteral(nestedField.type(), literalExpr); + if (value == null) { + return null; + } valueList.add(value); } if (inExpr.isNotIn()) { @@ -220,16 +230,62 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) { } } - if (expression != null && expression instanceof Unbound) { - try { - ((Unbound) expression).bind(schema.asStruct(), true); - return expression; - } catch (Exception e) { - LOG.warn("Failed to check expression: " + e.getMessage()); - return null; + return checkConversion(expression, schema); + } + + private static Expression checkConversion(Expression expression, Schema schema) { + if (expression == null) { + return null; + } + switch (expression.op()) { + case AND: { + And andExpr = (And) expression; + Expression left = checkConversion(andExpr.left(), schema); + Expression right = checkConversion(andExpr.right(), schema); + if (left != null && right != null) { + return andExpr; + } else if (left != null) { + return left; + } else if (right != null) { + return right; + } else { + return null; + } } + case OR: { + Or orExpr = (Or) expression; + Expression left = checkConversion(orExpr.left(), schema); + Expression right = checkConversion(orExpr.right(), schema); + if (left == null || right == null) { + return null; + } else { + return orExpr; + } + } + case NOT: { + Not notExpr = (Not) expression; + Expression child = checkConversion(notExpr.child(), schema); + if (child == null) { + return null; + } else { + return notExpr; + } + } + case TRUE: + case FALSE: + return expression; + default: + if (!(expression instanceof Unbound)) { + return null; + } + try { + ((Unbound) expression).bind(schema.asStruct(), true); + return expression; + } catch (Exception e) { + LOG.debug("Failed to check expression: {}", e.getMessage()); + return null; + } } - return null; } public static Object extractDorisLiteral(org.apache.iceberg.types.Type icebergType, Expr expr) { @@ -248,6 +304,7 @@ public static Object extractDorisLiteral(org.apache.iceberg.types.Type icebergTy DateLiteral dateLiteral = (DateLiteral) expr; switch (icebergTypeID) { case STRING: + case DATE: return dateLiteral.getStringValue(); case TIMESTAMP: return dateLiteral.unixTimestamp(TimeUtils.getTimeZone()) * MILLIS_TO_NANO_TIME; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index e2564eae527015..961fb8ae1d6679 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -40,6 +40,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileType; @@ -50,6 +51,7 @@ import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CombinedScanTask; @@ -90,6 +92,7 @@ public class IcebergScanNode extends FileQueryScanNode { private IcebergSource source; private Table icebergTable; + private List pushdownIcebergPredicates = Lists.newArrayList(); /** * External file scan node for Query iceberg table @@ -180,7 +183,6 @@ public List getSplits() throws UserException { } private List doGetSplits() throws UserException { - TableScan scan = icebergTable.newScan(); // set snapshot @@ -199,6 +201,7 @@ private List doGetSplits() throws UserException { } for (Expression predicate : expressions) { scan = scan.filter(predicate); + this.pushdownIcebergPredicates.add(predicate.toString()); } // get splits @@ -439,4 +442,17 @@ protected void toThrift(TPlanNode planNode) { } } } + + @Override + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (pushdownIcebergPredicates.isEmpty()) { + return super.getNodeExplainString(prefix, detailLevel); + } + StringBuilder sb = new StringBuilder(); + for (String predicate : pushdownIcebergPredicates) { + sb.append(prefix).append(prefix).append(predicate).append("\n"); + } + return super.getNodeExplainString(prefix, detailLevel) + + String.format("%sicebergPredicatePushdown=\n%s\n", prefix, sb); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java new file mode 100644 index 00000000000000..f1471cb1ec47dd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.BoolLiteral; +import org.apache.doris.analysis.CompoundPredicate; +import org.apache.doris.analysis.CompoundPredicate.Operator; +import org.apache.doris.analysis.DateLiteral; +import org.apache.doris.analysis.DecimalLiteral; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FloatLiteral; +import org.apache.doris.analysis.InPredicate; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import org.apache.iceberg.Schema; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class IcebergPredicateTest { + + public static Schema schema; + + @BeforeClass + public static void before() throws AnalysisException { + schema = new Schema( + Types.NestedField.required(1, "c_int", Types.IntegerType.get()), + Types.NestedField.required(2, "c_long", Types.LongType.get()), + Types.NestedField.required(3, "c_bool", Types.BooleanType.get()), + Types.NestedField.required(4, "c_float", Types.FloatType.get()), + Types.NestedField.required(5, "c_double", Types.DoubleType.get()), + Types.NestedField.required(6, "c_dec", Types.DecimalType.of(20, 10)), + Types.NestedField.required(7, "c_date", Types.DateType.get()), + Types.NestedField.required(8, "c_ts", Types.TimestampType.withoutZone()), + Types.NestedField.required(10, "c_str", Types.StringType.get()) + ); + } + + @Test + public void testBinaryPredicate() throws AnalysisException { + List literalList = new ArrayList() {{ + add(new BoolLiteral(true)); + add(new DateLiteral("2023-01-02", Type.DATEV2)); + add(new DateLiteral("2024-01-02 12:34:56.123456", Type.DATETIMEV2)); + add(new DecimalLiteral(new BigDecimal("1.23"))); + add(new FloatLiteral(1.23, Type.FLOAT)); + add(new FloatLiteral(3.456, Type.DOUBLE)); + add(new IntLiteral(1, Type.TINYINT)); + add(new IntLiteral(1, Type.SMALLINT)); + add(new IntLiteral(1, Type.INT)); + add(new IntLiteral(1, Type.BIGINT)); + add(new StringLiteral("abc")); + add(new StringLiteral("2023-01-02")); + add(new StringLiteral("2023-01-02 01:02:03.456789")); + }}; + + List slotRefs = new ArrayList() {{ + add(new SlotRef(new TableName(), "c_int")); + add(new SlotRef(new TableName(), "c_long")); + add(new SlotRef(new TableName(), "c_bool")); + add(new SlotRef(new TableName(), "c_float")); + add(new SlotRef(new TableName(), "c_double")); + add(new SlotRef(new TableName(), "c_dec")); + add(new SlotRef(new TableName(), "c_date")); + add(new SlotRef(new TableName(), "c_ts")); + add(new SlotRef(new TableName(), "c_str")); + }}; + + // true indicates support for pushdown + Boolean[][] expects = new Boolean[][] { + { // int + false, false, false, false, false, false, true, true, true, true, false, false, false + }, + { // long + false, false, false, false, false, false, true, true, true, true, false, false, false + }, + { // boolean + true, false, false, false, false, false, false, false, false, false, false, false, false + }, + { // float + false, false, false, false, true, false, true, true, true, true, false, false, false + }, + { // double + false, false, false, true, true, true, true, true, true, true, false, false, false + }, + { // decimal + false, false, false, true, true, true, true, true, true, true, false, false, false + }, + { // date + false, true, false, false, false, false, true, true, true, true, false, true, false + }, + { // timestamp + false, true, true, false, false, false, false, false, false, true, false, false, false + }, + { // string + true, true, true, true, false, false, false, false, false, false, true, true, true + } + }; + + ArrayListMultimap validPredicateMap = ArrayListMultimap.create(); + + // binary predicate + for (int i = 0; i < slotRefs.size(); i++) { + final int loc = i; + List ret = literalList.stream().map(literal -> { + BinaryPredicate expr = new BinaryPredicate(BinaryPredicate.Operator.EQ, slotRefs.get(loc), literal); + Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); + validPredicateMap.put(expression != null, expr); + return expression != null; + }).collect(Collectors.toList()); + Assert.assertArrayEquals(expects[i], ret.toArray()); + } + + // in predicate + for (int i = 0; i < slotRefs.size(); i++) { + final int loc = i; + List ret = literalList.stream().map(literal -> { + InPredicate expr = new InPredicate(slotRefs.get(loc), Lists.newArrayList(literal), false); + Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); + validPredicateMap.put(expression != null, expr); + return expression != null; + }).collect(Collectors.toList()); + Assert.assertArrayEquals(expects[i], ret.toArray()); + } + + // not in predicate + for (int i = 0; i < slotRefs.size(); i++) { + final int loc = i; + List ret = literalList.stream().map(literal -> { + InPredicate expr = new InPredicate(slotRefs.get(loc), Lists.newArrayList(literal), true); + Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); + validPredicateMap.put(expression != null, expr); + return expression != null; + }).collect(Collectors.toList()); + Assert.assertArrayEquals(expects[i], ret.toArray()); + } + + // bool literal + Expression trueExpr = IcebergUtils.convertToIcebergExpr(new BoolLiteral(true), schema); + Expression falseExpr = IcebergUtils.convertToIcebergExpr(new BoolLiteral(false), schema); + Assert.assertEquals(Expressions.alwaysTrue(), trueExpr); + Assert.assertEquals(Expressions.alwaysFalse(), falseExpr); + validPredicateMap.put(true, new BoolLiteral(true)); + validPredicateMap.put(true, new BoolLiteral(false)); + + List validExprs = validPredicateMap.get(true); + List invalidExprs = validPredicateMap.get(false); + // OR predicate + // both valid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < validExprs.size(); j++) { + CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR, + validExprs.get(i), validExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(orPredicate, schema); + Assert.assertNotNull("pred: " + orPredicate.toSql(), expression); + } + } + // both invalid + for (int i = 0; i < invalidExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR, + invalidExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(orPredicate, schema); + Assert.assertNull("pred: " + orPredicate.toSql(), expression); + } + } + // valid or invalid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR, + validExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(orPredicate, schema); + Assert.assertNull("pred: " + orPredicate.toSql(), expression); + } + } + + // AND predicate + // both valid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < validExprs.size(); j++) { + CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND, + validExprs.get(i), validExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(andPredicate, schema); + Assert.assertNotNull("pred: " + andPredicate.toSql(), expression); + } + } + // both invalid + for (int i = 0; i < invalidExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND, + invalidExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(andPredicate, schema); + Assert.assertNull("pred: " + andPredicate.toSql(), expression); + } + } + // valid and invalid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND, + validExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(andPredicate, schema); + Assert.assertNotNull("pred: " + andPredicate.toSql(), expression); + Assert.assertEquals(IcebergUtils.convertToIcebergExpr(validExprs.get(i), schema).toString(), + expression.toString()); + } + } + + // NOT predicate + // valid + for (int i = 0; i < validExprs.size(); i++) { + CompoundPredicate notPredicate = new CompoundPredicate(Operator.NOT, + validExprs.get(i), null); + Expression expression = IcebergUtils.convertToIcebergExpr(notPredicate, schema); + Assert.assertNotNull("pred: " + notPredicate.toSql(), expression); + } + // invalid + for (int i = 0; i < invalidExprs.size(); i++) { + CompoundPredicate notPredicate = new CompoundPredicate(Operator.NOT, + invalidExprs.get(i), null); + Expression expression = IcebergUtils.convertToIcebergExpr(notPredicate, schema); + Assert.assertNull("pred: " + notPredicate.toSql(), expression); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java deleted file mode 100644 index 80b1c62819bb97..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.iceberg; - -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.BoolLiteral; -import org.apache.doris.analysis.DateLiteral; -import org.apache.doris.analysis.DecimalLiteral; -import org.apache.doris.analysis.FloatLiteral; -import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.LiteralExpr; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.StringLiteral; -import org.apache.doris.analysis.TableName; -import org.apache.doris.catalog.Type; -import org.apache.doris.common.AnalysisException; - -import org.apache.iceberg.Schema; -import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -public class TestIcebergPredict { - - public static Schema schema; - - @BeforeClass - public static void before() throws AnalysisException { - schema = new Schema( - Types.NestedField.required(1, "c_int", Types.IntegerType.get()), - Types.NestedField.required(2, "c_long", Types.LongType.get()), - Types.NestedField.required(3, "c_bool", Types.BooleanType.get()), - Types.NestedField.required(4, "c_float", Types.FloatType.get()), - Types.NestedField.required(5, "c_double", Types.DoubleType.get()), - Types.NestedField.required(6, "c_dec", Types.DecimalType.of(20, 10)), - Types.NestedField.required(7, "c_date", Types.DateType.get()), - Types.NestedField.required(8, "c_ts", Types.TimestampType.withoutZone()), - Types.NestedField.required(10, "c_str", Types.StringType.get()) - ); - } - - @Test - public void testBinaryPredicate() throws AnalysisException { - List literalList = new ArrayList() {{ - add(new BoolLiteral(true)); - add(new DateLiteral("2023-01-02", Type.DATEV2)); - add(new DateLiteral("2024-01-02 12:34:56.123456", Type.DATETIMEV2)); - add(new DecimalLiteral(new BigDecimal("1.23"))); - add(new FloatLiteral(1.23, Type.FLOAT)); - add(new FloatLiteral(3.456, Type.DOUBLE)); - add(new IntLiteral(1, Type.TINYINT)); - add(new IntLiteral(1, Type.SMALLINT)); - add(new IntLiteral(1, Type.INT)); - add(new IntLiteral(1, Type.BIGINT)); - add(new StringLiteral("abc")); - add(new StringLiteral("2023-01-02")); - add(new StringLiteral("2023-01-02 01:02:03.456789")); - }}; - - List slotRefs = new ArrayList() {{ - add(new SlotRef(new TableName(), "c_int")); - add(new SlotRef(new TableName(), "c_long")); - add(new SlotRef(new TableName(), "c_bool")); - add(new SlotRef(new TableName(), "c_float")); - add(new SlotRef(new TableName(), "c_double")); - add(new SlotRef(new TableName(), "c_dec")); - add(new SlotRef(new TableName(), "c_date")); - add(new SlotRef(new TableName(), "c_ts")); - add(new SlotRef(new TableName(), "c_str")); - }}; - - // true indicates support for pushdown - Boolean[][] expects = new Boolean[][] { - { // int - false, false, false, false, false, false, true, true, true, true, false, false, false - }, - { // long - false, false, false, false, false, false, true, true, true, true, false, false, false - }, - { // boolean - true, false, false, false, false, false, false, false, false, false, false, false, false - }, - { // float - false, false, false, false, true, false, true, true, true, true, false, false, false - }, - { // double - false, false, false, true, true, true, true, true, true, true, false, false, false - }, - { // decimal - false, false, false, true, true, true, true, true, true, true, false, false, false - }, - { // date - false, false, false, false, false, false, true, true, true, true, false, true, false - }, - { // timestamp - false, true, true, false, false, false, false, false, false, true, false, false, false - }, - { // string - true, true, true, true, false, false, false, false, false, false, true, true, true - } - }; - - for (int i = 0; i < slotRefs.size(); i++) { - final int loc = i; - List ret = literalList.stream().map(literal -> { - BinaryPredicate expr = new BinaryPredicate(BinaryPredicate.Operator.EQ, slotRefs.get(loc), literal); - Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); - return expression != null; - }).collect(Collectors.toList()); - Assert.assertArrayEquals(expects[i], ret.toArray()); - } - } -} diff --git a/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out b/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out new file mode 100644 index 00000000000000..0569e0d01d8375 --- /dev/null +++ b/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out @@ -0,0 +1,611 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +11801003 35210325 + +-- !q02 -- +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 + +-- !q03 -- +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL + +-- !q04 -- +1992-01-02 REG AIR +1992-01-02 SHIP +1992-01-03 REG AIR +1992-01-03 TRUCK +1992-01-04 AIR +1992-01-04 FOB +1992-01-04 RAIL +1992-01-04 REG AIR +1992-01-04 TRUCK +1992-01-05 AIR + +-- !q04 -- +2023-03-07T20:35:59.064 +2023-03-07T20:35:59.087 +2023-03-07T20:35:59.110 +2023-03-07T20:35:59.129 +2023-03-07T20:35:59.224 + diff --git a/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy new file mode 100644 index 00000000000000..58518489ef0e38 --- /dev/null +++ b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_predicate_conversion", "p2,external,hive,external_remote,external_remote_hive") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + + sql """drop catalog if exists test_iceberg_predicate_conversion;""" + sql """ + create catalog if not exists test_iceberg_predicate_conversion properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + + sql """switch test_iceberg_predicate_conversion;""" + sql """ use `iceberg_catalog`; """ + + def sqlstr = """select glue_int, glue_varchar from iceberg_glue_types where glue_varchar > date '2023-03-07' """ + order_qt_q01 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """ref(name="glue_varchar") > "2023-03-07 00:00:00""" + } + + sqlstr = """select l_shipdate from lineitem where l_shipdate in ("1997-05-18", "1996-05-06"); """ + order_qt_q02 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """ref(name="l_shipdate") in""" + contains """"1997-05-18"""" + contains """"1996-05-06"""" + } + + sqlstr = """select l_shipdate, l_shipmode from lineitem where l_shipdate in ("1997-05-18", "1996-05-06") and l_shipmode = "MAIL";""" + order_qt_q03 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """ref(name="l_shipdate") in""" + contains """"1997-05-18"""" + contains """"1996-05-06"""" + contains """ref(name="l_shipmode") == "MAIL"""" + } + + sqlstr = """select l_shipdate, l_shipmode from lineitem where l_shipdate in ("1997-05-18", "1996-05-06") or NOT(l_shipmode = "MAIL") order by l_shipdate, l_shipmode limit 10""" + plan = """(ref(name="l_shipdate") in ("1997-05-18", "1996-05-06") or not(ref(name="l_shipmode") == "MAIL"))""" + order_qt_q04 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """or not(ref(name="l_shipmode") == "MAIL"))""" + contains """ref(name="l_shipdate")""" + contains """"1997-05-18"""" + contains """"1996-05-06"""" + } + + sqlstr = """select glue_timstamp from iceberg_glue_types where glue_timstamp > '2023-03-07 20:35:59' order by glue_timstamp limit 5""" + order_qt_q04 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """ref(name="glue_timstamp") > 1678192559000000""" + } + } +}