diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java index 96eeb76684eae4..7f626655442b09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java @@ -128,11 +128,11 @@ private TablePartitionValues getPartitionValues() { private TablePartitionValues loadPartitionValues(MaxComputeSchemaCacheValue schemaCacheValue) { List partitionSpecs = schemaCacheValue.getPartitionSpecs(); List partitionTypes = schemaCacheValue.getPartitionTypes(); + List partitionColumnNames = schemaCacheValue.getPartitionColumnNames(); TablePartitionValues partitionValues = new TablePartitionValues(); partitionValues.addPartitions(partitionSpecs, partitionSpecs.stream() - .map(p -> parsePartitionValues(getPartitionColumns().stream().map(c -> c.getName()).collect( - Collectors.toList()), p)) + .map(p -> parsePartitionValues(partitionColumnNames, p)) .collect(Collectors.toList()), partitionTypes); return partitionValues; @@ -187,9 +187,21 @@ public Optional initSchema() { } List partitionColumns = odpsTable.getSchema().getPartitionColumns(); + List partitionColumnNames = new ArrayList<>(partitionColumns.size()); + List partitionTypes = new ArrayList<>(partitionColumns.size()); - for (com.aliyun.odps.Column partitionColumn : partitionColumns) { - columnNameToOdpsColumn.put(partitionColumn.getName(), partitionColumn); + // sort partition columns to align partitionTypes and partitionName. + List partitionDorisColumns = new ArrayList<>(); + for (com.aliyun.odps.Column partColumn : partitionColumns) { + Type partitionType = mcTypeToDorisType(partColumn.getTypeInfo()); + Column dorisCol = new Column(partColumn.getName(), partitionType, true, null, + true, partColumn.getComment(), true, -1); + + columnNameToOdpsColumn.put(partColumn.getName(), partColumn); + partitionColumnNames.add(partColumn.getName()); + partitionDorisColumns.add(dorisCol); + partitionTypes.add(partitionType); + schema.add(dorisCol); } List partitionSpecs; @@ -200,22 +212,9 @@ public Optional initSchema() { } else { partitionSpecs = ImmutableList.of(); } - // sort partition columns to align partitionTypes and partitionName. - Map partitionNameToColumns = Maps.newHashMap(); - for (com.aliyun.odps.Column partColumn : partitionColumns) { - Column dorisCol = new Column(partColumn.getName(), - mcTypeToDorisType(partColumn.getTypeInfo()), true, null, - true, partColumn.getComment(), true, -1); - partitionNameToColumns.put(dorisCol.getName(), dorisCol); - } - List partitionTypes = partitionNameToColumns.values() - .stream() - .map(Column::getType) - .collect(Collectors.toList()); - schema.addAll(partitionNameToColumns.values()); - return Optional.of(new MaxComputeSchemaCacheValue(schema, odpsTable, partitionSpecs, partitionNameToColumns, - partitionTypes)); + return Optional.of(new MaxComputeSchemaCacheValue(schema, odpsTable, partitionColumnNames, + partitionSpecs, partitionDorisColumns, partitionTypes)); } private Type mcTypeToDorisType(TypeInfo typeInfo) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java index b8337d96120a5f..0d0fb69a3e4e0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeSchemaCacheValue.java @@ -22,36 +22,35 @@ import org.apache.doris.datasource.SchemaCacheValue; import com.aliyun.odps.Table; -import com.google.common.collect.Lists; import lombok.Getter; import lombok.Setter; import java.util.List; -import java.util.Map; -import java.util.Set; @Getter @Setter public class MaxComputeSchemaCacheValue extends SchemaCacheValue { private Table odpsTable; + private List partitionColumnNames; private List partitionSpecs; - private Map partitionNameToColumns; + private List partitionColumns; private List partitionTypes; - public MaxComputeSchemaCacheValue(List schema, Table odpsTable, List partitionSpecs, - Map partitionNameToColumns, List partitionTypes) { + public MaxComputeSchemaCacheValue(List schema, Table odpsTable, List partitionColumnNames, + List partitionSpecs, List partitionColumns, List partitionTypes) { super(schema); this.odpsTable = odpsTable; this.partitionSpecs = partitionSpecs; - this.partitionNameToColumns = partitionNameToColumns; + this.partitionColumnNames = partitionColumnNames; + this.partitionColumns = partitionColumns; this.partitionTypes = partitionTypes; } - public Set getPartitionColNames() { - return partitionNameToColumns.keySet(); + public List getPartitionColumns() { + return partitionColumns; } - public List getPartitionColumns() { - return Lists.newArrayList(partitionNameToColumns.values()); + public List getPartitionColumnNames() { + return partitionColumnNames; } } diff --git a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out index 63677f14720494..c388e4eddf658a 100644 --- a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out +++ b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out @@ -77,11 +77,30 @@ true 9 900 9000 1002 Sample data 2 2023-08-02 1005 Sample data 5 2023-08-05 +-- !q8 -- +mc_bigint bigint Yes true \N +mc_string text Yes true \N +dt text Yes true \N + -- !replay_q6 -- 1003 Sample data 3 2023-08-03 1004 Sample data 4 2023-08-04 1005 Sample data 5 2023-08-05 +-- !multi_partition_q1 -- +city text Yes true \N +gender boolean Yes true \N +mnt smallint Yes true \N +order_rate float Yes true \N +amount decimal(24,9) Yes true \N +cut_date date Yes true \N +create_time datetime(3) Yes true \N +finished_time datetime(6) Yes true \N +yy text Yes true \N +mm text Yes true \N +dd text Yes true \N +pt bigint Yes true \N + -- !multi_partition_q1 -- yy=2023/mm=08/dd=01/pt=1 yy=2023/mm=08/dd=02/pt=2 @@ -129,8 +148,6 @@ yy=2023/mm=08/dd=05/pt=5 -- !multi_partition_q8 -- 6 --- !multi_partition_q9 -- - -- !multi_partition_q10 -- 3 2023 08 03 3 2023 08 03 @@ -187,3 +204,47 @@ dt=2023-08-03 dt=2023-08-04 dt=2023-08-05 +-- !part2_q1 -- +1 2024-01-01 Y +2 2024-04-04 N +3 2027-01-09 Y +4 2024-01-01 N + +-- !part2_q2 -- +1 2024-01-01 Y + +-- !part2_q3 -- +1 2024-01-01 Y +3 2027-01-09 Y + +-- !part2_q4 -- +1 2024-01-01 Y +2 2024-04-04 N +4 2024-01-01 N + +-- !part2_q5 -- +2024-01-01 +2024-01-01 +2024-04-04 + +-- !part2_q6 -- +2024-01-01 N 4 +2024-01-01 Y 1 +2024-04-04 N 2 + +-- !part2_q7 -- +N 2024-01-01 2024-01-01 4 4 4 +N 2024-04-04 2024-04-04 2 2 2 +Y 2024-01-01 2024-01-01 1 1 1 +Y 2027-01-09 2027-01-09 3 3 3 + +-- !part2_q8 -- +N +N +Y + +-- !part2_q9 -- +id text Yes true \N +ds text Yes true \N +audit_flag text Yes true \N + diff --git a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy index 81133270fb6e60..6843a586d86d01 100644 --- a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy +++ b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy @@ -314,8 +314,17 @@ INSERT INTO `other_db_mc_parts` PARTITION (dt='e') VALUES (1005, 'Sample data 5'); - - + CREATE TABLE `mc_parts2` ( + `id` STRING COMMENT 'Id' + ) + PARTITIONED BY ( + `ds` STRING , + `audit_flag` STRING + ); + INSERT INTO TABLE `mc_parts2` PARTITION (`ds`='2024-01-01', `audit_flag`='Y') values ('1'); + INSERT INTO TABLE `mc_parts2` PARTITION (`ds`='2024-04-04', `audit_flag`='N') values ('2'); + INSERT INTO TABLE `mc_parts2` PARTITION (`ds`='2027-01-09', `audit_flag`='Y') values ('3'); + INSERT INTO TABLE `mc_parts2` PARTITION (`ds`='2024-01-01', `audit_flag`='N') values ('4'); */ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remote,external_remote_maxcompute") { String enabled = context.config.otherConfigs.get("enableMaxComputeTest") @@ -353,6 +362,7 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot order_qt_q5 """ select * from mc_parts where dt > '2023-08-03' order by mc_bigint """ order_qt_q6 """ select * from mc_parts where dt > '2023-08-03' and mc_bigint > 1002 """ order_qt_q7 """ select * from mc_parts where dt < '2023-08-03' or (mc_bigint > 1003 and dt > '2023-08-04') order by mc_bigint, dt; """ + qt_q8 """ desc mc_parts """ } sql """ switch `${mc_catalog_name}`; """ @@ -380,6 +390,8 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot sql """ refresh catalog ${mc_catalog_name} """ sql """ switch `${mc_catalog_name}`; """ sql """ use `${mc_db}`; """ + + qt_multi_partition_q1 """ desc multi_partitions """ order_qt_multi_partition_q1 """ show partitions from multi_partitions; """ order_qt_multi_partition_q2 """ select pt, create_time, yy, mm, dd from multi_partitions where pt>-1 and yy > '' and mm > '' and dd >'' order by pt , dd; """ order_qt_multi_partition_q3 """ select sum(pt), create_time, yy, mm, dd from multi_partitions where yy > '' and mm > '' and dd >'' group by create_time, yy, mm, dd order by create_time,dd ; """ @@ -425,5 +437,15 @@ suite("test_external_catalog_maxcompute", "p2,external,maxcompute,external_remot order_qt_show_partition """ show partitions from mc_parts """ + order_qt_part2_q1 """ select * from mc_parts2 """ + order_qt_part2_q2 """ SELECT * FROM `mc_parts2` WHERE `ds` = '2024-01-01' AND `audit_flag` = 'Y'; """ + order_qt_part2_q3 """ SELECT * FROM `mc_parts2` WHERE `audit_flag` = 'Y';""" + order_qt_part2_q4 """ SELECT * FROM `mc_parts2` WHERE `ds` BETWEEN '2024-01-01' AND '2027-01-01';""" + order_qt_part2_q5 """ SELECT ds FROM `mc_parts2` WHERE `ds` != '2027-01-09';""" + order_qt_part2_q6 """ SELECT ds,audit_flag,id FROM `mc_parts2` WHERE `ds` != '2027-01-09';""" + order_qt_part2_q7 """ SELECT audit_flag,ds,ds,id,id,id FROM `mc_parts2`;""" + order_qt_part2_q8 """ SELECT audit_flag FROM `mc_parts2` WHERE `ds` != '2027-01-09';""" + qt_part2_q9 """ desc mc_parts2 """ + } }