Skip to content

branch-3.1: [fix](maxcompute)Fix the NereidsException caused by non-existent table columns when pushing down mc predicates. #55635 #55746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 1 commit into from
Sep 8, 2025
Merged

branch-3.1: [fix](maxcompute)Fix the NereidsException caused by non-existent table columns when pushing down mc predicates. #55635 #55746

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions ...-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.doris.datasource.maxcompute.source;

import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.CompoundPredicate.Operator;
import org.apache.doris.analysis.DateLiteral;
Expand Down Expand Up @@ -302,7 +301,7 @@ protected void convertPredicate() {
for (Expr dorisPredicate : conjuncts) {
try {
odpsPredicates.add(convertExprToOdpsPredicate(dorisPredicate));
} catch (AnalysisException e) {
} catch (Exception e) {
Log.warn("Failed to convert predicate " + dorisPredicate.toString() + "Reason: "
+ e.getMessage());
}
Expand Down Expand Up @@ -362,11 +361,13 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException
: com.aliyun.odps.table.optimizer.predicate.InPredicate.Operator.NOT_IN;

String columnName = convertSlotRefToColumnName(expr.getChild(0));
if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
throw new AnalysisException("Column " + columnName + " not found in table, can not push "
+ "down predicate to MaxCompute " + table.getName());
}
com.aliyun.odps.OdpsType odpsType = table.getColumnNameToOdpsColumn().get(columnName).getType();

StringBuilder stringBuilder = new StringBuilder();


stringBuilder.append(columnName);
stringBuilder.append(" ");
stringBuilder.append(odpsOp.getDescription());
Expand Down Expand Up @@ -420,6 +421,10 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException

if (odpsOp != null) {
String columnName = convertSlotRefToColumnName(expr.getChild(0));
if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
throw new AnalysisException("Column " + columnName + " not found in table, can not push "
+ "down predicate to MaxCompute " + table.getName());
}
com.aliyun.odps.OdpsType odpsType = table.getColumnNameToOdpsColumn().get(columnName).getType();
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(columnName);
Expand Down Expand Up @@ -455,10 +460,6 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException
private String convertSlotRefToColumnName(Expr expr) throws AnalysisException {
if (expr instanceof SlotRef) {
return ((SlotRef) expr).getColumnName();
} else if (expr instanceof CastExpr) {
if (expr.getChild(0) instanceof SlotRef) {
return ((SlotRef) expr.getChild(0)).getColumnName();
}
}

throw new AnalysisException("Do not support convert ["
Expand Down
18 changes: 18 additions & 0 deletions regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,3 +248,21 @@ id text Yes true \N
ds text Yes true \N
audit_flag text Yes true \N

-- !where_1 --
1 WS0001 Example Web Site 1 Example Company 1234 Main Street Apt Unit 101 Metropolis, NY USA 1
2 WS0002 Example Web Site 2 Tech Innovations 5678 Tech Avenue Suite Suite 200 Gotham, CA USA 1
3 WS0003 Example Web Site 3 Health Corp 9101 Health Drive Floor Floor 3 Star City, TX USA 1
4 WS0004 Example Web Site 4 Edu Tech 1122 Education Lane Building Building 1 Smallville, FL USA 1
5 WS0005 Example Web Site 5 Travel Inc 3344 Tourist Street Unit Unit 5 Metropolis, WA USA 1

-- !where_2 --

-- !where_3 --

-- !where_4 --
1 WS0001 2023-01-01 2023-12-31 Example Web Site 1 20230101 20231231 E-commerce John Doe 101 Retail Online retail website Jane Smith 201 Example Company 1234 Main Street Apt Unit 101 Metropolis County NY 12345 USA -5.0 8.25
2 WS0002 2023-02-01 2023-11-30 Example Web Site 2 20230201 20231130 Technology Alice Johnson 102 Tech Tech news and reviews Bob Brown 202 Tech Innovations 5678 Tech Avenue Suite Suite 200 Gotham County CA 67890 USA -8.0 7.50
3 WS0003 2023-03-01 2023-10-31 Example Web Site 3 20230301 20231031 Healthcare Robert White 103 Health Healthcare services and products Emily Green 203 Health Corp 9101 Health Drive Floor Floor 3 Star City County TX 23456 USA -6.0 6.75
4 WS0004 2023-04-01 2023-09-30 Example Web Site 4 20230401 20230930 Education David Black 104 EdTech Educational technology platform Fiona Grey 204 Edu Tech 1122 Education Lane Building Building 1 Smallville County FL 34567 USA -4.0 5.00
5 WS0005 2023-05-01 2023-08-31 Example Web Site 5 20230501 20230831 Travel Sophia Blue 105 Travel Travel and booking services Daniel Red 205 Travel Inc 3344 Tourist Street Unit Unit 5 Metropolis County WA 45678 USA -7.0 8.00

9 changes: 9 additions & 0 deletions regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,5 +447,14 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot
order_qt_part2_q8 """ SELECT audit_flag FROM `mc_parts2` WHERE `ds` != '2027-01-09';"""
qt_part2_q9 """ desc mc_parts2 """


qt_where_1 """ WITH active_us_sites AS (SELECT web_site_sk, web_site_id, web_name, web_open_date_sk, web_close_date_sk, web_company_name, web_city,
web_state, web_country, CONCAT_WS(' ', web_street_number, web_street_name, web_street_type, web_suite_number) AS full_address, COUNT(1) OVER (PARTITION BY web_site_id) AS a FROM web_site )
SELECT web_site_sk, web_site_id, web_name, web_company_name, full_address, CONCAT(web_city, ', ', web_state) AS city_state,
web_country , a FROM active_us_sites where a>=1 ORDER BY web_site_sk;"""

qt_where_2 """select * from web_site where web_mkt_id > 100000000000000 ORDER BY web_site_sk;"""
qt_where_3 """ select * from web_site where web_close_date_sk > CURRENT_DATE() ORDER BY web_site_sk; """
qt_where_4 """ select * from web_site where web_rec_end_date < CURRENT_DATE() ORDER BY web_site_sk; """
}
}