diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java index d9d167e786b7c6..09f052f2cea235 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java @@ -66,8 +66,6 @@ public MaxComputeExternalTable(long id, String name, String remoteName, MaxCompu super(id, name, remoteName, catalog, db, TableType.MAX_COMPUTE_EXTERNAL_TABLE); } - private Map columnNameToOdpsColumn = new HashMap(); - @Override protected synchronized void makeSureInitialized() { super.makeSureInitialized(); @@ -166,7 +164,10 @@ private static List parsePartitionValues(List partitionColumns, } public Map getColumnNameToOdpsColumn() { - return columnNameToOdpsColumn; + makeSureInitialized(); + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((MaxComputeSchemaCacheValue) value).getColumnNameToOdpsColumn()) + .orElse(Collections.emptyMap()); } @Override @@ -179,7 +180,7 @@ public Optional initSchema() { TableIdentifier tableIdentifier = mcCatalog.getOdpsTableIdentifier(dbName, name); List columns = odpsTable.getSchema().getColumns(); - + Map columnNameToOdpsColumn = new HashMap<>(); for (com.aliyun.odps.Column column : columns) { columnNameToOdpsColumn.put(column.getName(), column); } @@ -218,7 +219,7 @@ public Optional initSchema() { } return Optional.of(new MaxComputeSchemaCacheValue(schema, odpsTable, tableIdentifier, - partitionColumnNames, partitionSpecs, partitionDorisColumns, partitionTypes)); + partitionColumnNames, partitionSpecs, partitionDorisColumns, partitionTypes, columnNameToOdpsColumn)); } private Type mcTypeToDorisType(TypeInfo typeInfo) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java index cf87725c680618..cd734985e6e92b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java @@ -27,6 +27,7 @@ import lombok.Setter; import java.util.List; +import java.util.Map; @Getter @Setter @@ -37,10 +38,11 @@ public class MaxComputeSchemaCacheValue extends SchemaCacheValue { private List partitionSpecs; private List partitionColumns; private List partitionTypes; + private Map columnNameToOdpsColumn; public MaxComputeSchemaCacheValue(List schema, Table odpsTable, TableIdentifier tableIdentifier, List partitionColumnNames, List partitionSpecs, List partitionColumns, - List partitionTypes) { + List partitionTypes, Map columnNameToOdpsColumn) { super(schema); this.odpsTable = odpsTable; this.tableIdentifier = tableIdentifier; @@ -48,6 +50,7 @@ public MaxComputeSchemaCacheValue(List schema, Table odpsTable, TableIde this.partitionColumnNames = partitionColumnNames; this.partitionColumns = partitionColumns; this.partitionTypes = partitionTypes; + this.columnNameToOdpsColumn = columnNameToOdpsColumn; } public List getPartitionColumns() { @@ -57,4 +60,8 @@ public List getPartitionColumns() { public List getPartitionColumnNames() { return partitionColumnNames; } + + public Map getColumnNameToOdpsColumn() { + return columnNameToOdpsColumn; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index e923b600d23dad..dc6ced6ee0113a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -62,6 +62,8 @@ import com.aliyun.odps.table.read.split.impl.IndexedInputSplit; import jline.internal.Log; import lombok.Setter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -88,6 +90,8 @@ public class MaxComputeScanNode extends FileQueryScanNode { static final DateTimeFormatter dateTime3Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); static final DateTimeFormatter dateTime6Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS"); + private static final Logger LOG = LogManager.getLogger(MaxComputeScanNode.class); + private final MaxComputeExternalTable table; private Predicate filterPredicate; List requiredPartitionColumns = new ArrayList<>(); @@ -355,6 +359,9 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException String columnName = convertSlotRefToColumnName(expr.getChild(0)); if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) { + Map columnMap = table.getColumnNameToOdpsColumn(); + LOG.warn("ColumnNameToOdpsColumn size=" + columnMap.size() + + ", keys=[" + String.join(", ", columnMap.keySet()) + "]"); throw new AnalysisException("Column " + columnName + " not found in table, can not push " + "down predicate to MaxCompute " + table.getName()); } @@ -415,6 +422,9 @@ private Predicate convertExprToOdpsPredicate(Expr expr) throws AnalysisException if (odpsOp != null) { String columnName = convertSlotRefToColumnName(expr.getChild(0)); if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) { + Map columnMap = table.getColumnNameToOdpsColumn(); + LOG.warn("ColumnNameToOdpsColumn size=" + columnMap.size() + + ", keys=[" + String.join(", ", columnMap.keySet()) + "]"); throw new AnalysisException("Column " + columnName + " not found in table, can not push " + "down predicate to MaxCompute " + table.getName()); }