diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index cd9049f70e1630..c8f24db0da9464 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.maxcompute.source; import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.CompoundPredicate.Operator; import org.apache.doris.analysis.DateLiteral; @@ -302,7 +301,7 @@ protected void convertPredicate() { for (Expr dorisPredicate : conjuncts) { try { odpsPredicates.add(convertExprToOdpsPredicate(dorisPredicate)); - } catch (AnalysisException e) { + } catch (Exception e) { Log.warn("Failed to convert predicate " + dorisPredicate.toString() + "Reason: " + e.getMessage()); } @@ -362,11 +361,13 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException : com.aliyun.odps.table.optimizer.predicate.InPredicate.Operator.NOT_IN; String columnName = convertSlotRefToColumnName(expr.getChild(0)); + if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) { + throw new AnalysisException("Column " + columnName + " not found in table, can not push " + + "down predicate to MaxCompute " + table.getName()); + } com.aliyun.odps.OdpsType odpsType = table.getColumnNameToOdpsColumn().get(columnName).getType(); StringBuilder stringBuilder = new StringBuilder(); - - stringBuilder.append(columnName); stringBuilder.append(" "); stringBuilder.append(odpsOp.getDescription()); @@ -420,6 +421,10 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException if (odpsOp != null) { String columnName = convertSlotRefToColumnName(expr.getChild(0)); + if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) { + throw new AnalysisException("Column " + columnName + " not found in table, can not push " + + "down predicate to MaxCompute " + table.getName()); + } com.aliyun.odps.OdpsType odpsType = table.getColumnNameToOdpsColumn().get(columnName).getType(); StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append(columnName); @@ -455,10 +460,6 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException private String convertSlotRefToColumnName(Expr expr) throws AnalysisException { if (expr instanceof SlotRef) { return ((SlotRef) expr).getColumnName(); - } else if (expr instanceof CastExpr) { - if (expr.getChild(0) instanceof SlotRef) { - return ((SlotRef) expr.getChild(0)).getColumnName(); - } } throw new AnalysisException("Do not support convert [" diff --git a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out index c388e4eddf658a..af7b8dd3f120a5 100644 --- a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out +++ b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out @@ -248,3 +248,21 @@ id text Yes true \N ds text Yes true \N audit_flag text Yes true \N +-- !where_1 -- +1 WS0001 Example Web Site 1 Example Company 1234 Main Street Apt Unit 101 Metropolis, NY USA 1 +2 WS0002 Example Web Site 2 Tech Innovations 5678 Tech Avenue Suite Suite 200 Gotham, CA USA 1 +3 WS0003 Example Web Site 3 Health Corp 9101 Health Drive Floor Floor 3 Star City, TX USA 1 +4 WS0004 Example Web Site 4 Edu Tech 1122 Education Lane Building Building 1 Smallville, FL USA 1 +5 WS0005 Example Web Site 5 Travel Inc 3344 Tourist Street Unit Unit 5 Metropolis, WA USA 1 + +-- !where_2 -- + +-- !where_3 -- + +-- !where_4 -- +1 WS0001 2023-01-01 2023-12-31 Example Web Site 1 20230101 20231231 E-commerce John Doe 101 Retail Online retail website Jane Smith 201 Example Company 1234 Main Street Apt Unit 101 Metropolis County NY 12345 USA -5.0 8.25 +2 WS0002 2023-02-01 2023-11-30 Example Web Site 2 20230201 20231130 Technology Alice Johnson 102 Tech Tech news and reviews Bob Brown 202 Tech Innovations 5678 Tech Avenue Suite Suite 200 Gotham County CA 67890 USA -8.0 7.50 +3 WS0003 2023-03-01 2023-10-31 Example Web Site 3 20230301 20231031 Healthcare Robert White 103 Health Healthcare services and products Emily Green 203 Health Corp 9101 Health Drive Floor Floor 3 Star City County TX 23456 USA -6.0 6.75 +4 WS0004 2023-04-01 2023-09-30 Example Web Site 4 20230401 20230930 Education David Black 104 EdTech Educational technology platform Fiona Grey 204 Edu Tech 1122 Education Lane Building Building 1 Smallville County FL 34567 USA -4.0 5.00 +5 WS0005 2023-05-01 2023-08-31 Example Web Site 5 20230501 20230831 Travel Sophia Blue 105 Travel Travel and booking services Daniel Red 205 Travel Inc 3344 Tourist Street Unit Unit 5 Metropolis County WA 45678 USA -7.0 8.00 + diff --git a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy index 6843a586d86d01..827ed963873fdd 100644 --- a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy +++ b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy @@ -447,5 +447,14 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot order_qt_part2_q8 """ SELECT audit_flag FROM `mc_parts2` WHERE `ds` != '2027-01-09';""" qt_part2_q9 """ desc mc_parts2 """ + + qt_where_1 """ WITH active_us_sites AS (SELECT web_site_sk, web_site_id, web_name, web_open_date_sk, web_close_date_sk, web_company_name, web_city, + web_state, web_country, CONCAT_WS(' ', web_street_number, web_street_name, web_street_type, web_suite_number) AS full_address, COUNT(1) OVER (PARTITION BY web_site_id) AS a FROM web_site ) + SELECT web_site_sk, web_site_id, web_name, web_company_name, full_address, CONCAT(web_city, ', ', web_state) AS city_state, + web_country , a FROM active_us_sites where a>=1 ORDER BY web_site_sk;""" + + qt_where_2 """select * from web_site where web_mkt_id > 100000000000000 ORDER BY web_site_sk;""" + qt_where_3 """ select * from web_site where web_close_date_sk > CURRENT_DATE() ORDER BY web_site_sk; """ + qt_where_4 """ select * from web_site where web_rec_end_date < CURRENT_DATE() ORDER BY web_site_sk; """ } }