From 747b120a4391a7cdb8c715390b35a71563f15d03 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 10 Jul 2025 15:56:24 +0800 Subject: [PATCH 01/43] intro data_lake_partition_values --- gensrc/thrift/PlanNodes.thrift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 247b7dd43be9b2..1802fa7f5fd17e 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -463,7 +463,9 @@ struct TFileScanRangeParams { // This is used to represent the latest id. 25: optional i64 current_schema_id; // All schema information used in the current query process - 26: optional list history_schema_info + 26: optional list history_schema_info + // partition values for data lake table format like iceberg/hudi/paimon/lakesoul + 27: optional map data_lake_partition_values; } struct TFileRangeDesc { From 902fc81c5df23eafd40a252ea79aff375f3f62a0 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 15 Jul 2025 22:40:00 +0800 Subject: [PATCH 02/43] get partitionValues --- .../iceberg/source/IcebergScanNode.java | 21 +++++++++++++------ .../iceberg/source/IcebergSplit.java | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index bc9f1ee8842694..4e30047ff9f764 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -60,20 +60,22 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.Snapshot; -import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.TableScanUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -312,11 +314,6 @@ private CloseableIterable planFileScanTask(TableScan scan) { } private Split createIcebergSplit(FileScanTask fileScanTask) { - if (isPartitionedTable) { - StructLike structLike = fileScanTask.file().partition(); - // Counts the number of partitions read - partitionPathSet.add(structLike.toString()); - } String originalPath = fileScanTask.file().path().toString(); LocationPath locationPath = LocationPath.of(originalPath, source.getCatalog().getCatalogProperty().getStoragePropertiesMap()); @@ -335,6 +332,18 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { } split.setTableFormatType(TableFormatType.ICEBERG); split.setTargetSplitSize(targetSplitSize); + if (isPartitionedTable) { + PartitionData partitionData = (PartitionData) fileScanTask.file().partition(); + Map partitionValues = new HashMap<>(); + List fileds = partitionData.getPartitionType().asNestedType().fields(); + for (int i = 0; i < fileds.size(); i++) { + NestedField field = fileds.get(i); + Object value = partitionData.get(i); + partitionValues.put(field.name().toLowerCase(), value == null ? null : value.toString()); + } + // Counts the number of partitions read + partitionPathSet.add(partitionData.toString()); + } return split; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java index 67a3b3d37ff357..453ca60dfa617b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java @@ -41,6 +41,8 @@ public class IcebergSplit extends FileSplit { private Map config; // tableLevelRowCount will be set only table-level count push down opt is available. private long tableLevelRowCount = -1; + // Partition values are used to do runtime filter partition pruning. + private Map partitionValues = null; // File path will be changed if the file is modified, so there's no need to get modification time. public IcebergSplit(LocationPath file, long start, long length, long fileLength, String[] hosts, From 82911e2b9b3112b0ee07ce97dd9f632523029912 Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 16 Jul 2025 17:19:40 +0800 Subject: [PATCH 03/43] impl be --- be/src/vec/exec/scan/file_scanner.cpp | 90 ++++++++++++------- be/src/vec/exec/scan/file_scanner.h | 15 +++- .../iceberg/source/IcebergScanNode.java | 1 + gensrc/thrift/PlanNodes.thrift | 4 +- 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 715a71974d0e59..ccaa5ea1224e47 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -206,8 +206,7 @@ Status FileScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conjun bool FileScanner::_check_partition_prune_expr(const VExprSPtr& expr) { if (expr->is_slot_ref()) { auto* slot_ref = static_cast(expr.get()); - return _partition_slot_index_map.find(slot_ref->slot_id()) != - _partition_slot_index_map.end(); + return _partition_col_names.find(slot_ref->expr_name()) != _partition_col_names.end(); } if (expr->is_literal()) { return true; @@ -242,16 +241,19 @@ void FileScanner::_init_runtime_filter_partition_prune_block() { } } -Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_all) { +Status FileScanner::_process_runtime_filters_partition_prune( + const std::unordered_map>& + partition_col_descs, + bool& can_filter_all) { SCOPED_TIMER(_runtime_filter_partition_prune_timer); - if (_runtime_filter_partition_prune_ctxs.empty() || _partition_col_descs.empty()) { + if (_runtime_filter_partition_prune_ctxs.empty() || partition_col_descs.empty()) { return Status::OK(); } size_t partition_value_column_size = 1; // 1. Get partition key values to string columns. - std::unordered_map parititon_slot_id_to_column; - for (auto const& partition_col_desc : _partition_col_descs) { + std::unordered_map partition_slot_id_to_column; + for (auto const& partition_col_desc : partition_col_descs) { const auto& [partition_value, partition_slot_desc] = partition_col_desc.second; auto test_serde = partition_slot_desc->get_data_type_ptr()->get_serde(); auto partition_value_column = partition_slot_desc->get_data_type_ptr()->create_column(); @@ -260,7 +262,7 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al uint64_t num_deserialized = 0; RETURN_IF_ERROR(test_serde->deserialize_column_from_fixed_json( *col_ptr, slice, partition_value_column_size, &num_deserialized, {})); - parititon_slot_id_to_column[partition_slot_desc->id()] = std::move(partition_value_column); + partition_slot_id_to_column[partition_slot_desc->id()] = std::move(partition_value_column); } // 2. Fill _runtime_filter_partition_prune_block from the partition column, then execute conjuncts and filter block. @@ -272,10 +274,10 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al // should be ignored from reading continue; } - if (parititon_slot_id_to_column.find(slot_desc->id()) != - parititon_slot_id_to_column.end()) { + if (partition_slot_id_to_column.find(slot_desc->id()) != + partition_slot_id_to_column.end()) { auto data_type = slot_desc->get_data_type_ptr(); - auto partition_value_column = std::move(parititon_slot_id_to_column[slot_desc->id()]); + auto partition_value_column = std::move(partition_slot_id_to_column[slot_desc->id()]); if (data_type->is_nullable()) { _runtime_filter_partition_prune_block.insert( index, ColumnWithTypeAndName( @@ -913,26 +915,31 @@ Status FileScanner::_get_next_reader() { const TFileRangeDesc& range = _current_range; _current_range_path = range.path; - if (!_partition_slot_descs.empty()) { - // we need get partition columns first for runtime filter partition pruning - RETURN_IF_ERROR(_generate_parititon_columns()); + // try to get the partition columns from the range + RETURN_IF_ERROR(_generate_partition_columns()); + RETURN_IF_ERROR(_generate_data_lake_partition_columns()); - if (_state->query_options().enable_runtime_filter_partition_prune) { - // if enable_runtime_filter_partition_prune is true, we need to check whether this range can be filtered out - // by runtime filter partition prune - if (_push_down_conjuncts.size() < _conjuncts.size()) { - // there are new runtime filters, need to re-init runtime filter partition pruning ctxs - _init_runtime_filter_partition_prune_ctxs(); - } + const auto& partition_col_descs = !_partition_col_descs.empty() + ? _partition_col_descs + : _data_lake_partition_col_descs; - bool can_filter_all = false; - RETURN_IF_ERROR(_process_runtime_filters_partition_prune(can_filter_all)); - if (can_filter_all) { - // this range can be filtered out by runtime filter partition pruning - // so we need to skip this range - COUNTER_UPDATE(_runtime_filter_partition_pruned_range_counter, 1); - continue; - } + if (_state->query_options().enable_runtime_filter_partition_prune && + !partition_col_descs.empty()) { + // if enable_runtime_filter_partition_prune is true, we need to check whether this range can be filtered out + // by runtime filter partition prune + if (_push_down_conjuncts.size() < _conjuncts.size()) { + // there are new runtime filters, need to re-init runtime filter partition pruning ctxs + _init_runtime_filter_partition_prune_ctxs(); + } + + bool can_filter_all = false; + RETURN_IF_ERROR( + _process_runtime_filters_partition_prune(partition_col_descs, can_filter_all)); + if (can_filter_all) { + // this range can be filtered out by runtime filter partition pruning + // so we need to skip this range + COUNTER_UPDATE(_runtime_filter_partition_pruned_range_counter, 1); + continue; } } @@ -1393,7 +1400,7 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, const ExternalFileMappingInfo& external_info, int64_t* init_reader_ms, int64_t* get_block_ms) { _current_range = range; - RETURN_IF_ERROR(_generate_parititon_columns()); + RETURN_IF_ERROR(_generate_partition_columns()); TFileFormatType::type format_type = _get_current_format_type(); Status init_status = Status::OK(); @@ -1455,7 +1462,7 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, return Status::OK(); } -Status FileScanner::_generate_parititon_columns() { +Status FileScanner::_generate_partition_columns() { _partition_col_descs.clear(); const TFileRangeDesc& range = _current_range; if (range.__isset.columns_from_path && !_partition_slot_descs.empty()) { @@ -1480,6 +1487,22 @@ Status FileScanner::_generate_parititon_columns() { return Status::OK(); } +Status FileScanner::_generate_data_lake_partition_columns() { + _data_lake_partition_col_descs.clear(); + if (_current_range.__isset.data_lake_partition_values) { + const auto& partition_values = _current_range.data_lake_partition_values; + for (const auto& [key, value] : partition_values) { + if (_col_name_to_slot_id->find(key) == _col_name_to_slot_id->end()) { + return Status::InternalError("Unknown data lake partition column, col_name={}", + key); + } + const auto* slot_desc = _file_slot_descs[_col_name_to_slot_id->at(key)]; + _data_lake_partition_col_descs.emplace(key, std::make_tuple(value, slot_desc)); + } + } + return Status::OK(); +} + Status FileScanner::_generate_missing_columns() { _missing_col_descs.clear(); if (!_missing_cols.empty()) { @@ -1523,10 +1546,17 @@ Status FileScanner::_init_expr_ctxes() { if (!key_map.empty()) { for (size_t i = 0; i < key_map.size(); i++) { partition_name_to_key_index_map.emplace(key_map[i], i); + _partition_col_names.insert(key_map[i]); } } } + if (_current_range.__isset.data_lake_partition_values) { + for (const auto& partition_value : _current_range.data_lake_partition_values) { + _partition_col_names.insert(partition_value.first); + } + } + _num_of_columns_from_file = _params->num_of_columns_from_file; for (const auto& slot_info : _params->required_slots) { diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h index 99585585c20984..5e72ba63cadd3e 100644 --- a/be/src/vec/exec/scan/file_scanner.h +++ b/be/src/vec/exec/scan/file_scanner.h @@ -192,6 +192,13 @@ class FileScanner : public Scanner { _partition_col_descs; std::unordered_map _missing_col_descs; + // store data lake partition column descriptors + std::unordered_map> + _data_lake_partition_col_descs; + + // partition column names, used for initializing runtime filter partition prune context + std::set _partition_col_names; + // idx of skip_bitmap_col in _input_tuple_desc int32_t _skip_bitmap_col_idx {-1}; int32_t _sequence_map_col_uid {-1}; @@ -241,12 +248,16 @@ class FileScanner : public Scanner { Status _convert_to_output_block(Block* block); Status _truncate_char_or_varchar_columns(Block* block); void _truncate_char_or_varchar_column(Block* block, int idx, int len); - Status _generate_parititon_columns(); + Status _generate_partition_columns(); + Status _generate_data_lake_partition_columns(); Status _generate_missing_columns(); bool _check_partition_prune_expr(const VExprSPtr& expr); void _init_runtime_filter_partition_prune_ctxs(); void _init_runtime_filter_partition_prune_block(); - Status _process_runtime_filters_partition_prune(bool& is_partition_pruned); + Status _process_runtime_filters_partition_prune( + const std::unordered_map>& + partition_col_descs, + bool& is_partition_pruned); Status _process_conjuncts_for_dict_filter(); Status _process_late_arrival_conjuncts(); void _get_slot_ids(VExpr* expr, std::vector* slot_ids); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 4e30047ff9f764..d3374351eff7f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -203,6 +203,7 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli } } tableFormatFileDesc.setIcebergParams(fileDesc); + rangeDesc.setDataLakePartitionValues(icebergSplit.getPartitionValues()); rangeDesc.setTableFormatParams(tableFormatFileDesc); } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 1802fa7f5fd17e..2010d2254455b7 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -464,8 +464,6 @@ struct TFileScanRangeParams { 25: optional i64 current_schema_id; // All schema information used in the current query process 26: optional list history_schema_info - // partition values for data lake table format like iceberg/hudi/paimon/lakesoul - 27: optional map data_lake_partition_values; } struct TFileRangeDesc { @@ -495,6 +493,8 @@ struct TFileRangeDesc { 12: optional string fs_name 13: optional TFileFormatType format_type; 14: optional i64 self_split_weight + // partition values for data lake table format like iceberg/hudi/paimon/lakesoul + 15: optional map data_lake_partition_values; } struct TSplitSource { From 4639b3b3067d34a045a0b0640329ca61f975140b Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 16 Jul 2025 19:34:35 +0800 Subject: [PATCH 04/43] parse partition value --- .../datasource/iceberg/IcebergUtils.java | 37 +++++++++++++++++++ .../iceberg/source/IcebergScanNode.java | 3 +- .../iceberg/source/IcebergSplit.java | 2 +- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 4c96eb91a20096..f4dbe25aa93831 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -602,6 +602,43 @@ public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type) { } } + private static final String HIVE_NULL = "__HIVE_DEFAULT_PARTITION__"; + + /** + * Convert Iceberg partition value to string. + * @param type + * @param value + * @return + */ + public static String toPartitionString(org.apache.iceberg.types.Type type, Object value) { + if (value == null) { + return HIVE_NULL; + } + + switch (type.typeId()) { + case BOOLEAN: + case INTEGER: + case LONG: + case FLOAT: + case DOUBLE: + case STRING: + case UUID: + case DECIMAL: + return value.toString(); + case FIXED: + case BINARY: + if (value instanceof byte[]) { + return new String((byte[]) value, java.nio.charset.StandardCharsets.UTF_8); + } + return value.toString(); + case DATE: + return value.toString(); + default: + throw new UnsupportedOperationException( + "Unsupported type for toPartitionString: " + type); + } + } + public static Table getIcebergTable(ExternalTable dorisTable) { return Env.getCurrentEnv() .getExtMetaCacheMgr() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d3374351eff7f3..018bae043df668 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -340,7 +340,8 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { for (int i = 0; i < fileds.size(); i++) { NestedField field = fileds.get(i); Object value = partitionData.get(i); - partitionValues.put(field.name().toLowerCase(), value == null ? null : value.toString()); + String partitionString = IcebergUtils.toPartitionString(field.type(), value); + partitionValues.put(field.name(), partitionString); } // Counts the number of partitions read partitionPathSet.add(partitionData.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java index 453ca60dfa617b..0d08f53b3e8525 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java @@ -42,7 +42,7 @@ public class IcebergSplit extends FileSplit { // tableLevelRowCount will be set only table-level count push down opt is available. private long tableLevelRowCount = -1; // Partition values are used to do runtime filter partition pruning. - private Map partitionValues = null; + private Map icebergPartitionValues = null; // File path will be changed if the file is modified, so there's no need to get modification time. public IcebergSplit(LocationPath file, long start, long length, long fileLength, String[] hosts, From cafde455784fc09e51a8de6762440f335937ca1d Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 16 Jul 2025 19:46:26 +0800 Subject: [PATCH 05/43] fix --- .../doris/datasource/iceberg/source/IcebergScanNode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 018bae043df668..cf2ab3dd93d286 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -203,7 +203,7 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli } } tableFormatFileDesc.setIcebergParams(fileDesc); - rangeDesc.setDataLakePartitionValues(icebergSplit.getPartitionValues()); + rangeDesc.setDataLakePartitionValues(icebergSplit.getIcebergPartitionValues()); rangeDesc.setTableFormatParams(tableFormatFileDesc); } @@ -343,6 +343,7 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { String partitionString = IcebergUtils.toPartitionString(field.type(), value); partitionValues.put(field.name(), partitionString); } + split.setIcebergPartitionValues(partitionValues); // Counts the number of partitions read partitionPathSet.add(partitionData.toString()); } From ebe076753d20dddef9587d6b44c14e77f2c7c19c Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 16 Jul 2025 20:35:04 +0800 Subject: [PATCH 06/43] fix bug --- be/src/vec/exec/scan/file_scanner.cpp | 5 +++-- be/src/vec/exec/scan/file_scanner.h | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index ccaa5ea1224e47..67ef95ae1ef777 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -1492,11 +1492,11 @@ Status FileScanner::_generate_data_lake_partition_columns() { if (_current_range.__isset.data_lake_partition_values) { const auto& partition_values = _current_range.data_lake_partition_values; for (const auto& [key, value] : partition_values) { - if (_col_name_to_slot_id->find(key) == _col_name_to_slot_id->end()) { + if (_all_col_name_to_slot_desc.find(key) == _all_col_name_to_slot_desc.end()) { return Status::InternalError("Unknown data lake partition column, col_name={}", key); } - const auto* slot_desc = _file_slot_descs[_col_name_to_slot_id->at(key)]; + const auto* slot_desc = _all_col_name_to_slot_desc[key]; _data_lake_partition_col_descs.emplace(key, std::make_tuple(value, slot_desc)); } } @@ -1583,6 +1583,7 @@ Status FileScanner::_init_expr_ctxes() { _partition_slot_index_map.emplace(slot_id, kit->second); } } + _all_col_name_to_slot_desc.emplace(it->second->col_name(), it->second); } // set column name to default value expr map diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h index 5e72ba63cadd3e..ddae84d97e89df 100644 --- a/be/src/vec/exec/scan/file_scanner.h +++ b/be/src/vec/exec/scan/file_scanner.h @@ -192,6 +192,8 @@ class FileScanner : public Scanner { _partition_col_descs; std::unordered_map _missing_col_descs; + // store all slot descriptors, used for initializing runtime filter partition prune context + std::unordered_map _all_col_name_to_slot_desc; // store data lake partition column descriptors std::unordered_map> _data_lake_partition_col_descs; From 07cb25cb72534c029327f2c70cb3a8ee0d021c97 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 17 Jul 2025 16:00:01 +0800 Subject: [PATCH 07/43] impl rf paritition prune for paimon --- .../apache/doris/datasource/paimon/PaimonUtil.java | 14 ++++++++++++++ .../datasource/paimon/source/PaimonScanNode.java | 10 ++++++++-- .../datasource/paimon/source/PaimonSplit.java | 10 ++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 8f82f3c3aa0695..4ca612b128f220 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -42,6 +42,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.options.ConfigOption; @@ -63,6 +64,7 @@ import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Projection; +import org.apache.paimon.utils.RowDataToObjectArrayConverter; import java.io.IOException; import java.util.ArrayList; @@ -380,4 +382,16 @@ public static String encodeObjectToString(T t) { } } + public static Map getPartitionInfoMap(Table table, BinaryRow partitionValues) { + Map partitionInfoMap = new HashMap<>(); + List partitionKeys = table.partitionKeys(); + RowType partitionType = table.rowType().project(partitionKeys); + RowDataToObjectArrayConverter toObjectArrayConverter = new RowDataToObjectArrayConverter( + partitionType); + Object[] partitionValuesArray = toObjectArrayConverter.convert(partitionValues); + for (int i = 0; i < partitionKeys.size(); i++) { + partitionInfoMap.put(partitionKeys.get(i), partitionValuesArray[i].toString()); + } + return partitionInfoMap; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 9eecaa20d35071..eefbfc064df4d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -243,6 +243,7 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) tableFormatFileDesc.setTableLevelRowCount(paimonSplit.getRowCount().get()); } tableFormatFileDesc.setPaimonParams(fileDesc); + rangeDesc.setDataLakePartitionValues(paimonSplit.getPaimonPartitionValues()); rangeDesc.setTableFormatParams(tableFormatFileDesc); } @@ -275,12 +276,15 @@ public List getSplits(int numBackends) throws UserException { BinaryRow partitionValue = dataSplit.partition(); selectedPartitionValues.add(partitionValue); + Map partitionInfoMap = PaimonUtil.getPartitionInfoMap( + source.getPaimonTable(), partitionValue); Optional> optRawFiles = dataSplit.convertToRawFiles(); Optional> optDeletionFiles = dataSplit.deletionFiles(); if (applyCountPushdown && dataSplit.mergedRowCountAvailable()) { splitStat.setMergedRowCount(dataSplit.mergedRowCount()); PaimonSplit split = new PaimonSplit(dataSplit); split.setRowCount(dataSplit.mergedRowCount()); + split.setPaimonPartitionValues(partitionInfoMap); pushDownCountSplits.add(split); pushDownCountSum += dataSplit.mergedRowCount(); } else if (!forceJniScanner && supportNativeReader(optRawFiles)) { @@ -305,10 +309,12 @@ public List getSplits(int numBackends) throws UserException { null, PaimonSplit.PaimonSplitCreator.DEFAULT); for (Split dorisSplit : dorisSplits) { - ((PaimonSplit) dorisSplit).setSchemaId(file.schemaId()); + PaimonSplit paimonSplit = (PaimonSplit) dorisSplit; + paimonSplit.setSchemaId(file.schemaId()); + paimonSplit.setPaimonPartitionValues(partitionInfoMap); // try to set deletion file if (optDeletionFiles.isPresent() && optDeletionFiles.get().get(i) != null) { - ((PaimonSplit) dorisSplit).setDeletionFile(optDeletionFiles.get().get(i)); + paimonSplit.setDeletionFile(optDeletionFiles.get().get(i)); splitStat.setHasDeletionVector(true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java index 1bb0a2cd3dadbd..00700c3827f458 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java @@ -27,6 +27,7 @@ import org.apache.paimon.table.source.DeletionFile; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -37,6 +38,7 @@ public class PaimonSplit extends FileSplit { private Optional optDeletionFile = Optional.empty(); private Optional optRowCount = Optional.empty(); private Optional schemaId = Optional.empty(); + private Map paimonPartitionValues = null; public PaimonSplit(DataSplit split) { super(DUMMY_PATH, 0, 0, 0, 0, null, null); @@ -101,6 +103,14 @@ public Long getSchemaId() { return schemaId.orElse(null); } + public void setPaimonPartitionValues(Map paimonPartitionValues) { + this.paimonPartitionValues = paimonPartitionValues; + } + + public Map getPaimonPartitionValues() { + return paimonPartitionValues; + } + public static class PaimonSplitCreator implements SplitCreator { static final PaimonSplitCreator DEFAULT = new PaimonSplitCreator(); From 6143731dd254c9b59a31e17d37f1bf7817469a2a Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 17 Jul 2025 16:44:40 +0800 Subject: [PATCH 08/43] impl rf partition prune for hudi --- .../apache/doris/datasource/hudi/source/HudiScanNode.java | 8 ++++++++ .../apache/doris/datasource/hudi/source/HudiSplit.java | 2 ++ 2 files changed, 10 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 5cc100d6a4cb86..ea48691b9dd450 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -74,6 +74,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -307,6 +308,7 @@ private void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { } } tableFormatFileDesc.setHudiParams(fileDesc); + rangeDesc.setDataLakePartitionValues(hudiSplit.getHudiPartitionValues()); rangeDesc.setTableFormatParams(tableFormatFileDesc); } @@ -373,7 +375,12 @@ private void getPartitionSplits(HivePartition partition, List splits) thr new StoragePath(partition.getPath())); } + if (canUseNativeReader()) { + Map partitionValues = new HashMap<>(); + for (int i = 0; i < partition.getPartitionValues().size(); i++) { + partitionValues.put(partition.getColumns().get(i).getName(), partition.getPartitionValues().get(i)); + } fsView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { noLogsSplitNum.incrementAndGet(); String filePath = baseFile.getPath(); @@ -384,6 +391,7 @@ private void getPartitionSplits(HivePartition partition, List splits) thr HudiSplit hudiSplit = new HudiSplit(locationPath, 0, fileSize, fileSize, new String[0], partition.getPartitionValues()); hudiSplit.setTableFormatType(TableFormatType.HUDI); + hudiSplit.setHudiPartitionValues(partitionValues); splits.add(hudiSplit); }); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java index 2c3cbdb7fbac5c..9235bdde7a8836 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiSplit.java @@ -23,6 +23,7 @@ import lombok.Data; import java.util.List; +import java.util.Map; @Data public class HudiSplit extends FileSplit { @@ -40,4 +41,5 @@ public HudiSplit(LocationPath file, long start, long length, long fileLength, St private List hudiColumnNames; private List hudiColumnTypes; private List nestedFields; + private Map hudiPartitionValues; } From 13229a889c8e20ae37b68442aac976d94b601e92 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 17 Jul 2025 22:40:59 +0800 Subject: [PATCH 09/43] impl test_iceberg_partition_filter_partition_pruning --- .../iceberg/run18.sql | 101 +++++++++++++ ...rg_runtime_filter_partition_pruning.groovy | 141 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql new file mode 100644 index 00000000000000..cd02c106610ad4 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql @@ -0,0 +1,101 @@ +create database if not exists demo.partition_db; +use demo.partition_db; + +-- Partition by date type +CREATE TABLE date_partitioned ( + id BIGINT, + name STRING, + partition_key DATE +) USING ICEBERG +PARTITIONED BY (partition_key); + +-- Partition by integer type +CREATE TABLE int_partitioned ( + id BIGINT, + name STRING, + partition_key INT +) USING ICEBERG +PARTITIONED BY (partition_key); + +-- Partition by string type +CREATE TABLE string_partitioned ( + id BIGINT, + name STRING, + partition_key STRING +) USING ICEBERG +PARTITIONED BY (partition_key); + +-- Partition by timestamp type +CREATE TABLE timestamp_partitioned ( + id BIGINT, + name STRING, + partition_key TIMESTAMP +) USING ICEBERG +PARTITIONED BY (partition_key); + +-- Partition by boolean type +CREATE TABLE boolean_partitioned ( + id BIGINT, + name STRING, + partition_key BOOLEAN +) USING ICEBERG +PARTITIONED BY (partition_key); + +-- Partition by decimal type +CREATE TABLE decimal_partitioned ( + id BIGINT, + name STRING, + value FLOAT, + partition_key DECIMAL(10, 2) +) USING ICEBERG +PARTITIONED BY (partition_key); + +-- Insert data into date_partitioned table +INSERT INTO date_partitioned VALUES +(1, 'Alice', DATE '2024-01-01'), +(2, 'Bob', DATE '2024-01-01'), +(3, 'Charlie', DATE '2024-02-01'), +(4, 'David', DATE '2024-02-01'), +(5, 'Eve', DATE '2024-03-01'); + +-- Insert data into int_partitioned table +INSERT INTO int_partitioned VALUES +(1, 'Product A', 1), +(2, 'Product B', 1), +(3, 'Product C', 2), +(4, 'Product D', 2), +(5, 'Product E', 3); + +-- Insert data into string_partitioned table +INSERT INTO string_partitioned VALUES +(1, 'User1', 'North America'), +(2, 'User2', 'North America'), +(3, 'User3', 'Europe'), +(4, 'User4', 'Europe'), +(5, 'User5', 'Asia'), +(6, 'User6', 'Asia'); + +-- Insert data into timestamp_partitioned table +INSERT INTO timestamp_partitioned VALUES +(1, 'Event1', TIMESTAMP '2024-01-15 08:00:00'), +(2, 'Event2', TIMESTAMP '2024-01-15 09:00:00'), +(3, 'Event3', TIMESTAMP '2024-01-15 14:00:00'), +(4, 'Event4', TIMESTAMP '2024-01-16 10:00:00'), +(5, 'Event5', TIMESTAMP '2024-01-16 16:00:00'); + +-- Insert data into boolean_partitioned table +INSERT INTO boolean_partitioned VALUES +(1, 'Active User', true), +(2, 'Active Admin', true), +(3, 'Inactive User', false), +(4, 'Inactive Guest', false), +(5, 'Active Manager', true); + +-- Insert data into decimal_partitioned table +INSERT INTO decimal_partitioned VALUES +(1, 'Item A', 125.50, 10.50), +(2, 'Item B', 200.75, 10.50), +(3, 'Item C', 89.99, 25.25), +(4, 'Item D', 156.80, 25.25), +(5, 'Item E', 299.95, 50.00), +(6, 'Item F', 399.99, 50.00); \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy new file mode 100644 index 00000000000000..21ee5e916996a3 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_partition_filter_partition_pruning", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalog_name = "test_iceberg_systable_ctl" + String db_name = "demo.partition_db" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + sql """switch ${catalog_name}""" + sql """use ${db_name}""" + + def test_runtime_filter_partition_pruning = { + qt_runtime_filter_partition_pruning_decimal1 """ + select count(*) from decimal_partitioned where partition_key = + (select partition_key from decimal_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_decimal2 """ + select count(*) from decimal_partitioned where partition_key in + (select partition_key from decimal_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_decimal3 """ + select count(*) from decimal_partitioned where abs(partition_key) = + (select partition_key from decimal_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_int1 """ + select count(*) from int_partitioned where partition_key = + (select partition_key from int_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_int2 """ + select count(*) from int_partitioned where partition_key in + (select partition_key from int_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_int3 """ + select count(*) from int_partitioned where abs(partition_key) = + (select partition_key from int_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_string1 """ + select count(*) from string_partitioned where partition_key = + (select partition_key from string_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_string2 """ + select count(*) from string_partitioned where partition_key in + (select partition_key from string_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_date1 """ + select count(*) from date_partitioned where partition_key = + (select partition_key from date_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_date2 """ + select count(*) from date_partitioned where partition_key in + (select partition_key from date_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_timestamp1 """ + select count(*) from timestamp_partitioned where partition_key = + (select partition_key from timestamp_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_timestamp2 """ + select count(*) from timestamp_partitioned where partition_key in + (select partition_key from timestamp_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_boolean1 """ + select count(*) from boolean_partitioned where partition_key = + (select partition_key from boolean_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_boolean2 """ + select count(*) from boolean_partitioned where partition_key in + (select partition_key from boolean_partitioned + group by partition_key having count(*) > 0); + """ + } + try { + sql """ set enable_runtime_filter_partition_pruning = false; """ + test_runtime_filter_partition_pruning() + sql """ set enable_runtime_filter_partition_pruning = true; """ + test_runtime_filter_partition_pruning() + + } finally { + sql """ set enable_runtime_filter_partition_pruning = true; """ + } + +} From 13df2e3f83c8c2860441073a62cb2611d8ef7816 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 17 Jul 2025 22:54:59 +0800 Subject: [PATCH 10/43] fix --- ...eberg_runtime_filter_partition_pruning.out | 73 +++++++++++++++++++ ...rg_runtime_filter_partition_pruning.groovy | 32 ++++---- 2 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out new file mode 100644 index 00000000000000..977cdee1ab5339 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -0,0 +1,73 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !runtime_filter_partition_pruning_decimal1 -- +2 + +-- !runtime_filter_partition_pruning_decimal2 -- +4 + +-- !runtime_filter_partition_pruning_decimal3 -- +2 + +-- !runtime_filter_partition_pruning_int1 -- +1 + +-- !runtime_filter_partition_pruning_int2 -- +3 + +-- !runtime_filter_partition_pruning_int3 -- +1 + +-- !runtime_filter_partition_pruning_string1 -- +2 + +-- !runtime_filter_partition_pruning_string2 -- +4 + +-- !runtime_filter_partition_pruning_date1 -- +1 + +-- !runtime_filter_partition_pruning_date2 -- +3 + +-- !runtime_filter_partition_pruning_boolean1 -- +3 + +-- !runtime_filter_partition_pruning_boolean2 -- +5 + +-- !runtime_filter_partition_pruning_decimal1 -- +2 + +-- !runtime_filter_partition_pruning_decimal2 -- +4 + +-- !runtime_filter_partition_pruning_decimal3 -- +2 + +-- !runtime_filter_partition_pruning_int1 -- +1 + +-- !runtime_filter_partition_pruning_int2 -- +3 + +-- !runtime_filter_partition_pruning_int3 -- +1 + +-- !runtime_filter_partition_pruning_string1 -- +2 + +-- !runtime_filter_partition_pruning_string2 -- +4 + +-- !runtime_filter_partition_pruning_date1 -- +0 + +-- !runtime_filter_partition_pruning_date2 -- +0 + +-- !runtime_filter_partition_pruning_boolean1 -- +3 + +-- !runtime_filter_partition_pruning_boolean2 -- +5 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index 21ee5e916996a3..a24d1151ac647e 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -24,7 +24,7 @@ suite("test_iceberg_partition_filter_partition_pruning", "p0,external,doris,exte } String catalog_name = "test_iceberg_systable_ctl" - String db_name = "demo.partition_db" + String db_name = "partition_db" String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") String minio_port = context.config.otherConfigs.get("iceberg_minio_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -104,18 +104,18 @@ suite("test_iceberg_partition_filter_partition_pruning", "p0,external,doris,exte group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ - qt_runtime_filter_partition_pruning_timestamp1 """ - select count(*) from timestamp_partitioned where partition_key = - (select partition_key from timestamp_partitioned - group by partition_key having count(*) > 0 - order by partition_key desc limit 1); - """ - qt_runtime_filter_partition_pruning_timestamp2 """ - select count(*) from timestamp_partitioned where partition_key in - (select partition_key from timestamp_partitioned - group by partition_key having count(*) > 0 - order by partition_key desc limit 2); - """ + // qt_runtime_filter_partition_pruning_timestamp1 """ + // select count(*) from timestamp_partitioned where partition_key = + // (select partition_key from timestamp_partitioned + // group by partition_key having count(*) > 0 + // order by partition_key desc limit 1); + // """ + // qt_runtime_filter_partition_pruning_timestamp2 """ + // select count(*) from timestamp_partitioned where partition_key in + // (select partition_key from timestamp_partitioned + // group by partition_key having count(*) > 0 + // order by partition_key desc limit 2); + // """ qt_runtime_filter_partition_pruning_boolean1 """ select count(*) from boolean_partitioned where partition_key = (select partition_key from boolean_partitioned @@ -129,13 +129,13 @@ suite("test_iceberg_partition_filter_partition_pruning", "p0,external,doris,exte """ } try { - sql """ set enable_runtime_filter_partition_pruning = false; """ + sql """ set enable_runtime_filter_partition_prune = false; """ test_runtime_filter_partition_pruning() - sql """ set enable_runtime_filter_partition_pruning = true; """ + sql """ set enable_runtime_filter_partition_prune = true; """ test_runtime_filter_partition_pruning() } finally { - sql """ set enable_runtime_filter_partition_pruning = true; """ + sql """ set enable_runtime_filter_partition_prune = true; """ } } From a3f3334914075d38e82a0554c74698630c422de7 Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 18 Jul 2025 00:26:14 +0800 Subject: [PATCH 11/43] impl test_paimon_runtime_filter_partition_pruning --- .../paimon/run06.sql | 105 +++++++++++++ ...rg_runtime_filter_partition_pruning.groovy | 4 +- ...on_runtime_filter_partition_pruning.groovy | 139 ++++++++++++++++++ 3 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql new file mode 100644 index 00000000000000..98b0e55af3a2e8 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -0,0 +1,105 @@ +use paimon; + +create database if not exists partition_db; + +use partition_db; + +-- Partition by date type +CREATE TABLE date_partitioned ( + id BIGINT, + name STRING, + partition_key DATE, + PRIMARY KEY (id) NOT ENFORCED +) PARTITIONED BY (partition_key); + +-- Partition by integer type +CREATE TABLE int_partitioned ( + id BIGINT, + name STRING, + partition_key INT, + PRIMARY KEY (id) NOT ENFORCED +) PARTITIONED BY (partition_key); + +-- Partition by string type +CREATE TABLE string_partitioned ( + id BIGINT, + name STRING, + partition_key STRING, + PRIMARY KEY (id) NOT ENFORCED +) PARTITIONED BY (partition_key); + +-- Partition by timestamp type +CREATE TABLE timestamp_partitioned ( + id BIGINT, + name STRING, + partition_key TIMESTAMP, + PRIMARY KEY (id) NOT ENFORCED +) PARTITIONED BY (partition_key); + +-- Partition by boolean type +CREATE TABLE boolean_partitioned ( + id BIGINT, + name STRING, + partition_key BOOLEAN, + PRIMARY KEY (id) NOT ENFORCED +) PARTITIONED BY (partition_key); + +-- Partition by decimal type +CREATE TABLE decimal_partitioned ( + id BIGINT, + name STRING, + value FLOAT, + partition_key DECIMAL(10, 2), + PRIMARY KEY (id) NOT ENFORCED +) PARTITIONED BY (partition_key); + +-- Insert data into date_partitioned table +INSERT INTO date_partitioned VALUES +(1, 'Alice', DATE '2024-01-01'), +(2, 'Bob', DATE '2024-01-01'), +(3, 'Charlie', DATE '2024-02-01'), +(4, 'David', DATE '2024-02-01'), +(5, 'Eve', DATE '2024-03-01'); + +-- Insert data into int_partitioned table +INSERT INTO int_partitioned VALUES +(1, 'Product A', 1), +(2, 'Product B', 1), +(3, 'Product C', 2), +(4, 'Product D', 2), +(5, 'Product E', 3); + +-- Insert data into string_partitioned table +INSERT INTO string_partitioned VALUES +(1, 'User1', 'North America'), +(2, 'User2', 'North America'), +(3, 'User3', 'Europe'), +(4, 'User4', 'Europe'), +(5, 'User5', 'Asia'), +(6, 'User6', 'Asia'); + +-- Insert data into timestamp_partitioned table +INSERT INTO timestamp_partitioned VALUES +(1, 'Event1', TIMESTAMP '2024-01-15 08:00:00'), +(2, 'Event2', TIMESTAMP '2024-01-15 09:00:00'), +(3, 'Event3', TIMESTAMP '2024-01-15 14:00:00'), +(4, 'Event4', TIMESTAMP '2024-01-16 10:00:00'), +(5, 'Event5', TIMESTAMP '2024-01-16 16:00:00'); + +-- Insert data into boolean_partitioned table +INSERT INTO boolean_partitioned VALUES +(1, 'Active User', true), +(2, 'Active Admin', true), +(3, 'Inactive User', false), +(4, 'Inactive Guest', false), +(5, 'Active Manager', true); + +-- Insert data into decimal_partitioned table +INSERT INTO decimal_partitioned VALUES +(1, 'Item A', 125.50, 10.50), +(2, 'Item B', 200.75, 10.50), +(3, 'Item C', 89.99, 25.25), +(4, 'Item D', 156.80, 25.25), +(5, 'Item E', 299.95, 50.00), +(6, 'Item F', 399.99, 50.00); + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index a24d1151ac647e..5cad6e091711e8 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_iceberg_partition_filter_partition_pruning", "p0,external,doris,external_docker,external_docker_doris") { +suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,external_docker,external_docker_doris") { String enabled = context.config.otherConfigs.get("enableIcebergTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { @@ -23,7 +23,7 @@ suite("test_iceberg_partition_filter_partition_pruning", "p0,external,doris,exte return } - String catalog_name = "test_iceberg_systable_ctl" + String catalog_name = "test_iceberg_runtime_filter_partition_pruning" String db_name = "partition_db" String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") String minio_port = context.config.otherConfigs.get("iceberg_minio_port") diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy new file mode 100644 index 00000000000000..e65c23b12456a8 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String catalog_name = "test_paimon_runtime_filter_partition_pruning" + String db_name = "partition_db" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + + sql """drop catalog if exists ${catalog_name}""" + + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minio_port}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """use `${catalog_name}`.`${db_name}`;""" + + def test_runtime_filter_partition_pruning = { + qt_runtime_filter_partition_pruning_decimal1 """ + select count(*) from decimal_partitioned where partition_key = + (select partition_key from decimal_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_decimal2 """ + select count(*) from decimal_partitioned where partition_key in + (select partition_key from decimal_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_decimal3 """ + select count(*) from decimal_partitioned where abs(partition_key) = + (select partition_key from decimal_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_int1 """ + select count(*) from int_partitioned where partition_key = + (select partition_key from int_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_int2 """ + select count(*) from int_partitioned where partition_key in + (select partition_key from int_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_int3 """ + select count(*) from int_partitioned where abs(partition_key) = + (select partition_key from int_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_string1 """ + select count(*) from string_partitioned where partition_key = + (select partition_key from string_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_string2 """ + select count(*) from string_partitioned where partition_key in + (select partition_key from string_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_date1 """ + select count(*) from date_partitioned where partition_key = + (select partition_key from date_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_date2 """ + select count(*) from date_partitioned where partition_key in + (select partition_key from date_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_timestamp1 """ + select count(*) from timestamp_partitioned where partition_key = + (select partition_key from timestamp_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_timestamp2 """ + select count(*) from timestamp_partitioned where partition_key in + (select partition_key from timestamp_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_boolean1 """ + select count(*) from boolean_partitioned where partition_key = + (select partition_key from boolean_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_boolean2 """ + select count(*) from boolean_partitioned where partition_key in + (select partition_key from boolean_partitioned + group by partition_key having count(*) > 0); + """ + } + + try { + sql """ set enable_runtime_filter_partition_prune = false; """ + test_runtime_filter_partition_pruning() + sql """ set enable_runtime_filter_partition_prune = true; """ + test_runtime_filter_partition_pruning() + + } finally { + sql """ set enable_runtime_filter_partition_prune = true; """ + } + } +} + + From 8920afd21d06d2cd5f13c0354ee6b9c295225646 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 21 Jul 2025 11:16:03 +0800 Subject: [PATCH 12/43] flush paimon case --- .../paimon/run06.sql | 18 ++-- ...aimon_runtime_filter_partition_pruning.out | 85 +++++++++++++++++++ 2 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql index 98b0e55af3a2e8..c502a443b8323e 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -8,40 +8,35 @@ use partition_db; CREATE TABLE date_partitioned ( id BIGINT, name STRING, - partition_key DATE, - PRIMARY KEY (id) NOT ENFORCED + partition_key DATE ) PARTITIONED BY (partition_key); -- Partition by integer type CREATE TABLE int_partitioned ( id BIGINT, name STRING, - partition_key INT, - PRIMARY KEY (id) NOT ENFORCED + partition_key INT ) PARTITIONED BY (partition_key); -- Partition by string type CREATE TABLE string_partitioned ( id BIGINT, name STRING, - partition_key STRING, - PRIMARY KEY (id) NOT ENFORCED + partition_key STRING ) PARTITIONED BY (partition_key); -- Partition by timestamp type CREATE TABLE timestamp_partitioned ( id BIGINT, name STRING, - partition_key TIMESTAMP, - PRIMARY KEY (id) NOT ENFORCED + partition_key TIMESTAMP ) PARTITIONED BY (partition_key); -- Partition by boolean type CREATE TABLE boolean_partitioned ( id BIGINT, name STRING, - partition_key BOOLEAN, - PRIMARY KEY (id) NOT ENFORCED + partition_key BOOLEAN ) PARTITIONED BY (partition_key); -- Partition by decimal type @@ -49,8 +44,7 @@ CREATE TABLE decimal_partitioned ( id BIGINT, name STRING, value FLOAT, - partition_key DECIMAL(10, 2), - PRIMARY KEY (id) NOT ENFORCED + partition_key DECIMAL(10, 2) ) PARTITIONED BY (partition_key); -- Insert data into date_partitioned table diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out new file mode 100644 index 00000000000000..0fca62824b8319 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -0,0 +1,85 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !runtime_filter_partition_pruning_decimal1 -- +2 + +-- !runtime_filter_partition_pruning_decimal2 -- +4 + +-- !runtime_filter_partition_pruning_decimal3 -- +2 + +-- !runtime_filter_partition_pruning_int1 -- +1 + +-- !runtime_filter_partition_pruning_int2 -- +3 + +-- !runtime_filter_partition_pruning_int3 -- +1 + +-- !runtime_filter_partition_pruning_string1 -- +2 + +-- !runtime_filter_partition_pruning_string2 -- +4 + +-- !runtime_filter_partition_pruning_date1 -- +1 + +-- !runtime_filter_partition_pruning_date2 -- +3 + +-- !runtime_filter_partition_pruning_timestamp1 -- +1 + +-- !runtime_filter_partition_pruning_timestamp2 -- +2 + +-- !runtime_filter_partition_pruning_boolean1 -- +3 + +-- !runtime_filter_partition_pruning_boolean2 -- +5 + +-- !runtime_filter_partition_pruning_decimal1 -- +2 + +-- !runtime_filter_partition_pruning_decimal2 -- +4 + +-- !runtime_filter_partition_pruning_decimal3 -- +2 + +-- !runtime_filter_partition_pruning_int1 -- +1 + +-- !runtime_filter_partition_pruning_int2 -- +3 + +-- !runtime_filter_partition_pruning_int3 -- +1 + +-- !runtime_filter_partition_pruning_string1 -- +2 + +-- !runtime_filter_partition_pruning_string2 -- +4 + +-- !runtime_filter_partition_pruning_date1 -- +0 + +-- !runtime_filter_partition_pruning_date2 -- +0 + +-- !runtime_filter_partition_pruning_timestamp1 -- +0 + +-- !runtime_filter_partition_pruning_timestamp2 -- +0 + +-- !runtime_filter_partition_pruning_boolean1 -- +3 + +-- !runtime_filter_partition_pruning_boolean2 -- +5 + From ee59ecd5f75f12ff062e4b1e11a4609c23c23e54 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 21 Jul 2025 15:45:56 +0800 Subject: [PATCH 13/43] add test_hudi_runtime_filter_partition_pruning --- ...di_runtime_filter_partition_pruning.groovy | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy new file mode 100644 index 00000000000000..595dbf8b1f8ace --- /dev/null +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hudi_runtime_filter_partition_pruning", "p2,external,hudi,external_remote,external_remote_hudi") { + String enabled = context.config.otherConfigs.get("enableExternalHudiTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable hudi test") + return + } + + String catalog_name = "test_hudi_snapshot" + String props = context.config.otherConfigs.get("hudiEmrCatalog") + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + ${props} + ); + """ + + sql """ switch ${catalog_name};""" + sql """ use regression_hudi;""" + sql """ set enable_fallback_to_original_planner=false """ + + def test_runtime_filter_partition_pruning = { + // Test single partition table (INT type) + qt_runtime_filter_partition_pruning_one_partition_1 """ + select count(*) from one_partition_tb where part1 = + (select part1 from one_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_one_partition_2 """ + select count(*) from one_partition_tb where part1 in + (select part1 from one_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + qt_runtime_filter_partition_pruning_one_partition_3 """ + select count(*) from one_partition_tb where abs(part1) = + (select part1 from one_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + // Test two partition table (STRING + INT types) + qt_runtime_filter_partition_pruning_two_partition_1 """ + select count(*) from two_partition_tb where part1 = + (select part1 from two_partition_tb + group by part1, part2 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_two_partition_2 """ + select count(*) from two_partition_tb where part2 = + (select part2 from two_partition_tb + group by part1, part2 having count(*) > 0 + order by part2 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_two_partition_3 """ + select count(*) from two_partition_tb where part1 in + (select part1 from two_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Test three partition table (STRING + INT + STRING types) + qt_runtime_filter_partition_pruning_three_partition_1 """ + select count(*) from three_partition_tb where part1 = + (select part1 from three_partition_tb + group by part1, part2, part3 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_three_partition_2 """ + select count(*) from three_partition_tb where part2 = + (select part2 from three_partition_tb + group by part1, part2, part3 having count(*) > 0 + order by part2 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_three_partition_3 """ + select count(*) from three_partition_tb where part3 = + (select part3 from three_partition_tb + group by part1, part2, part3 having count(*) > 0 + order by part3 desc limit 1); + """ + + // Test BOOLEAN partition + qt_runtime_filter_partition_pruning_boolean_1 """ + select count(*) from boolean_partition_tb where part1 = + (select part1 from boolean_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_boolean_2 """ + select count(*) from boolean_partition_tb where part1 in + (select part1 from boolean_partition_tb + group by part1 having count(*) > 0); + """ + + // Test TINYINT partition + qt_runtime_filter_partition_pruning_tinyint_1 """ + select count(*) from tinyint_partition_tb where part1 = + (select part1 from tinyint_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_tinyint_2 """ + select count(*) from tinyint_partition_tb where part1 in + (select part1 from tinyint_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Test SMALLINT partition + qt_runtime_filter_partition_pruning_smallint_1 """ + select count(*) from smallint_partition_tb where part1 = + (select part1 from smallint_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_smallint_2 """ + select count(*) from smallint_partition_tb where part1 in + (select part1 from smallint_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Test INT partition + qt_runtime_filter_partition_pruning_int_1 """ + select count(*) from int_partition_tb where part1 = + (select part1 from int_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_int_2 """ + select count(*) from int_partition_tb where part1 in + (select part1 from int_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + qt_runtime_filter_partition_pruning_int_3 """ + select count(*) from int_partition_tb where abs(part1) = + (select part1 from int_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + // Test BIGINT partition + qt_runtime_filter_partition_pruning_bigint_1 """ + select count(*) from bigint_partition_tb where part1 = + (select part1 from bigint_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_bigint_2 """ + select count(*) from bigint_partition_tb where part1 in + (select part1 from bigint_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Test STRING partition + qt_runtime_filter_partition_pruning_string_1 """ + select count(*) from string_partition_tb where part1 = + (select part1 from string_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_string_2 """ + select count(*) from string_partition_tb where part1 in + (select part1 from string_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Test DATE partition + qt_runtime_filter_partition_pruning_date_1 """ + select count(*) from date_partition_tb where part1 = + (select part1 from date_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_date_2 """ + select count(*) from date_partition_tb where part1 in + (select part1 from date_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Test TIMESTAMP partition + qt_runtime_filter_partition_pruning_timestamp_1 """ + select count(*) from timestamp_partition_tb where part1 = + (select part1 from timestamp_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ + + qt_runtime_filter_partition_pruning_timestamp_2 """ + select count(*) from timestamp_partition_tb where part1 in + (select part1 from timestamp_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ + + // Additional complex scenarios with multiple filters + qt_runtime_filter_partition_pruning_complex_1 """ + select count(*) from three_partition_tb t1 + where t1.part1 in ( + select t2.part1 from three_partition_tb t2 + where t2.part2 = 2024 + group by t2.part1 having count(*) > 2 + ); + """ + + qt_runtime_filter_partition_pruning_complex_2 """ + select count(*) from two_partition_tb t1 + where t1.part1 = 'US' and t1.part2 in ( + select t2.part2 from two_partition_tb t2 + where t2.part1 = 'US' + group by t2.part2 having count(*) > 1 + ); + """ + + qt_runtime_filter_partition_pruning_complex_3 """ + select count(*) from three_partition_tb t1 + where (t1.part1, t1.part2) in ( + select t2.part1, t2.part2 from three_partition_tb t2 + where t2.part3 = 'Q1' + group by t2.part1, t2.part2 having count(*) > 0 + ); + """ + } + + try { + // Test with runtime filter partition pruning disabled + sql """ set enable_runtime_filter_partition_prune = false; """ + test_runtime_filter_partition_pruning() + + // Test with runtime filter partition pruning enabled + sql """ set enable_runtime_filter_partition_prune = true; """ + test_runtime_filter_partition_pruning() + + } finally { + // Restore default setting + sql """ set enable_runtime_filter_partition_prune = true; """ + } +} From 3f2374562c9529e5cd111f594c86310798ae18aa Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 21 Jul 2025 20:16:14 +0800 Subject: [PATCH 14/43] fix iceberg --- .../datasource/iceberg/IcebergUtils.java | 57 +++++++++++++------ .../iceberg/source/IcebergScanNode.java | 12 +--- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index f4dbe25aa93831..e3d0f19ed92670 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -78,6 +78,7 @@ import org.apache.iceberg.ManifestFile; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionsTable; @@ -119,6 +120,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -602,17 +604,26 @@ public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type) { } } - private static final String HIVE_NULL = "__HIVE_DEFAULT_PARTITION__"; + public static Map getPartitionInfoMap(PartitionData partitionData) { + Map partitionInfoMap = new HashMap<>(); + List fields = partitionData.getPartitionType().asNestedType().fields(); + for (int i = 0; i < fields.size(); i++) { + NestedField field = fields.get(i); + Object value = partitionData.get(i); + try { + String partitionString = serializePartitionValue(field.type(), value); + partitionInfoMap.put(field.name(), partitionString); + } catch (UnsupportedOperationException e) { + LOG.warn("Failed to serialize partition value for field {}: {}", field.name(), e.getMessage()); + return null; + } + } + return partitionInfoMap; + } - /** - * Convert Iceberg partition value to string. - * @param type - * @param value - * @return - */ - public static String toPartitionString(org.apache.iceberg.types.Type type, Object value) { + private static String serializePartitionValue(org.apache.iceberg.types.Type type, Object value) { if (value == null) { - return HIVE_NULL; + return "\\N"; } switch (type.typeId()) { @@ -625,17 +636,29 @@ public static String toPartitionString(org.apache.iceberg.types.Type type, Objec case UUID: case DECIMAL: return value.toString(); - case FIXED: - case BINARY: - if (value instanceof byte[]) { - return new String((byte[]) value, java.nio.charset.StandardCharsets.UTF_8); - } - return value.toString(); case DATE: - return value.toString(); + // Iceberg date is stored as days since epoch + LocalDateTime date = LocalDateTime.ofEpochSecond((Integer) value * 24 * 3600L, 0, + ZoneId.systemDefault().getRules().getOffset(Instant.now())); + return date.format(DateTimeFormatter.ISO_LOCAL_DATE); + case TIME: + // Iceberg time is stored as microseconds since midnight + long micros = (Long) value; + long seconds = micros / 1_000_000; + int nanos = (int) ((micros % 1_000_000) * 1000); + LocalDateTime time = LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0, 0, nanos); + time = time.plusSeconds(seconds); + return time.format(DateTimeFormatter.ISO_LOCAL_TIME); + case TIMESTAMP: + // Iceberg timestamp is stored as microseconds since epoch + long timestampMicros = (Long) value; + LocalDateTime timestamp = LocalDateTime.ofInstant( + Instant.ofEpochSecond(timestampMicros / 1_000_000, (int) (timestampMicros % 1_000_000) * 1000), + ZoneId.systemDefault()); + return timestamp.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: throw new UnsupportedOperationException( - "Unsupported type for toPartitionString: " + type); + "Unsupported type for serializePartitionValue: " + type); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index cf2ab3dd93d286..9fba9d7726925f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -68,14 +68,12 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.types.Conversions; -import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.TableScanUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -335,15 +333,7 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { split.setTargetSplitSize(targetSplitSize); if (isPartitionedTable) { PartitionData partitionData = (PartitionData) fileScanTask.file().partition(); - Map partitionValues = new HashMap<>(); - List fileds = partitionData.getPartitionType().asNestedType().fields(); - for (int i = 0; i < fileds.size(); i++) { - NestedField field = fileds.get(i); - Object value = partitionData.get(i); - String partitionString = IcebergUtils.toPartitionString(field.type(), value); - partitionValues.put(field.name(), partitionString); - } - split.setIcebergPartitionValues(partitionValues); + split.setIcebergPartitionValues(IcebergUtils.getPartitionInfoMap(partitionData)); // Counts the number of partitions read partitionPathSet.add(partitionData.toString()); } From b2cdade35a773fa9a5226e164b241ce7a15a1f28 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 21 Jul 2025 20:28:20 +0800 Subject: [PATCH 15/43] flush case --- ...eberg_runtime_filter_partition_pruning.out | 16 +++++++++++-- ...rg_runtime_filter_partition_pruning.groovy | 24 +++++++++---------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index 977cdee1ab5339..16dc58415ece1c 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -29,6 +29,12 @@ -- !runtime_filter_partition_pruning_date2 -- 3 +-- !runtime_filter_partition_pruning_timestamp1 -- +1 + +-- !runtime_filter_partition_pruning_timestamp2 -- +2 + -- !runtime_filter_partition_pruning_boolean1 -- 3 @@ -60,10 +66,16 @@ 4 -- !runtime_filter_partition_pruning_date1 -- -0 +1 -- !runtime_filter_partition_pruning_date2 -- -0 +3 + +-- !runtime_filter_partition_pruning_timestamp1 -- +1 + +-- !runtime_filter_partition_pruning_timestamp2 -- +2 -- !runtime_filter_partition_pruning_boolean1 -- 3 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index 5cad6e091711e8..c8344204c68153 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -104,18 +104,18 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ - // qt_runtime_filter_partition_pruning_timestamp1 """ - // select count(*) from timestamp_partitioned where partition_key = - // (select partition_key from timestamp_partitioned - // group by partition_key having count(*) > 0 - // order by partition_key desc limit 1); - // """ - // qt_runtime_filter_partition_pruning_timestamp2 """ - // select count(*) from timestamp_partitioned where partition_key in - // (select partition_key from timestamp_partitioned - // group by partition_key having count(*) > 0 - // order by partition_key desc limit 2); - // """ + qt_runtime_filter_partition_pruning_timestamp1 """ + select count(*) from timestamp_partitioned where partition_key = + (select partition_key from timestamp_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_timestamp2 """ + select count(*) from timestamp_partitioned where partition_key in + (select partition_key from timestamp_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ qt_runtime_filter_partition_pruning_boolean1 """ select count(*) from boolean_partitioned where partition_key = (select partition_key from boolean_partitioned From dbc7c546688a2365a973144dbc2fc10b42af6b8c Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 21 Jul 2025 21:05:13 +0800 Subject: [PATCH 16/43] fix paimon --- .../datasource/iceberg/IcebergUtils.java | 13 ++--- .../doris/datasource/paimon/PaimonUtil.java | 55 ++++++++++++++++++- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index e3d0f19ed92670..49f200cb05420f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -114,6 +114,7 @@ import java.nio.ByteBuffer; import java.time.DateTimeException; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Month; import java.time.ZoneId; @@ -638,16 +639,13 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type return value.toString(); case DATE: // Iceberg date is stored as days since epoch - LocalDateTime date = LocalDateTime.ofEpochSecond((Integer) value * 24 * 3600L, 0, - ZoneId.systemDefault().getRules().getOffset(Instant.now())); + LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME: // Iceberg time is stored as microseconds since midnight long micros = (Long) value; - long seconds = micros / 1_000_000; - int nanos = (int) ((micros % 1_000_000) * 1000); - LocalDateTime time = LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0, 0, nanos); - time = time.plusSeconds(seconds); + LocalDateTime time = LocalDateTime.ofEpochSecond(micros / 1_000_000, (int) (micros % 1_000_000) * 1000, + ZoneId.systemDefault().getRules().getOffset(Instant.now())); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP: // Iceberg timestamp is stored as microseconds since epoch @@ -657,8 +655,7 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type ZoneId.systemDefault()); return timestamp.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: - throw new UnsupportedOperationException( - "Unsupported type for serializePartitionValue: " + type); + throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 4ca612b128f220..b3365b30ee28a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -44,6 +44,7 @@ import org.apache.logging.log4j.Logger; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.partition.Partition; @@ -67,6 +68,11 @@ import org.apache.paimon.utils.RowDataToObjectArrayConverter; import java.io.IOException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Base64; import java.util.HashMap; @@ -390,8 +396,55 @@ public static Map getPartitionInfoMap(Table table, BinaryRow par partitionType); Object[] partitionValuesArray = toObjectArrayConverter.convert(partitionValues); for (int i = 0; i < partitionKeys.size(); i++) { - partitionInfoMap.put(partitionKeys.get(i), partitionValuesArray[i].toString()); + try { + String partitionValue = serializePartitionValue(partitionType.getFields().get(i).type(), + partitionValuesArray[i]); + partitionInfoMap.put(partitionKeys.get(i), partitionValue); + } catch (UnsupportedOperationException e) { + LOG.warn("Failed to serialize partition value for key {}: {}", partitionKeys.get(i), e.getMessage()); + return null; + } } return partitionInfoMap; } + + private static String serializePartitionValue(org.apache.paimon.types.DataType type, Object value) { + if (value == null) { + return "\\N"; + } + switch (type.getTypeRoot()) { + case BOOLEAN: + case INTEGER: + case BIGINT: + case FLOAT: + case DOUBLE: + case SMALLINT: + case TINYINT: + case DECIMAL: + case VARCHAR: + case CHAR: + return value.toString(); + case BINARY: + case VARBINARY: + return Base64.getEncoder().encodeToString((byte[]) value); + case DATE: + // Paimon date is stored as days since epoch + LocalDate date = LocalDate.ofEpochDay((Integer) value); + return date.format(DateTimeFormatter.ISO_LOCAL_DATE); + case TIME_WITHOUT_TIME_ZONE: + // Paimon time is stored as microseconds since midnight + long micros = (Long) value; + LocalDateTime time = LocalDateTime.ofEpochSecond(micros / 1_000_000, (int) (micros % 1_000_000) * 1000, + ZoneId.systemDefault().getRules().getOffset(Instant.now())); + return time.format(DateTimeFormatter.ISO_LOCAL_TIME); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + Timestamp timestamp = (Timestamp) value; + return timestamp.toLocalDateTime() + .atZone(ZoneId.systemDefault()) + .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + default: + throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); + } + } } From 3d69f152b704160ff1f4b52b756ea8b408d1d6d7 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 21 Jul 2025 21:13:50 +0800 Subject: [PATCH 17/43] flush out --- .../paimon/test_paimon_runtime_filter_partition_pruning.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index 0fca62824b8319..093cac83259e0e 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -66,10 +66,10 @@ 4 -- !runtime_filter_partition_pruning_date1 -- -0 +1 -- !runtime_filter_partition_pruning_date2 -- -0 +3 -- !runtime_filter_partition_pruning_timestamp1 -- 0 From 4a7f5a226b3b8657c557365ce0b4f07b22234bdc Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 15:05:27 +0800 Subject: [PATCH 18/43] fix timestamp with timezone problem --- .../doris/datasource/iceberg/IcebergUtils.java | 8 ++++---- .../iceberg/source/IcebergScanNode.java | 4 +++- .../doris/datasource/paimon/PaimonUtil.java | 16 ++++++++++------ .../datasource/paimon/source/PaimonScanNode.java | 7 +++++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 49f200cb05420f..f49a5fe1c3ddb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -642,17 +642,17 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME: - // Iceberg time is stored as microseconds since midnight + // Iceberg time is stored as microseconds since midnight without timezone long micros = (Long) value; LocalDateTime time = LocalDateTime.ofEpochSecond(micros / 1_000_000, (int) (micros % 1_000_000) * 1000, - ZoneId.systemDefault().getRules().getOffset(Instant.now())); + ZoneId.of("UTC").getRules().getOffset(Instant.now())); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP: - // Iceberg timestamp is stored as microseconds since epoch + // Iceberg timestamp is stored as microseconds since epoch without timezone long timestampMicros = (Long) value; LocalDateTime timestamp = LocalDateTime.ofInstant( Instant.ofEpochSecond(timestampMicros / 1_000_000, (int) (timestampMicros % 1_000_000) * 1000), - ZoneId.systemDefault()); + ZoneId.of("UTC")); return timestamp.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 9fba9d7726925f..14b507ac142410 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -333,7 +333,9 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { split.setTargetSplitSize(targetSplitSize); if (isPartitionedTable) { PartitionData partitionData = (PartitionData) fileScanTask.file().partition(); - split.setIcebergPartitionValues(IcebergUtils.getPartitionInfoMap(partitionData)); + if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { + split.setIcebergPartitionValues(IcebergUtils.getPartitionInfoMap(partitionData)); + } // Counts the number of partitions read partitionPathSet.add(partitionData.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index b3365b30ee28a1..330d737116c4f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -388,7 +388,7 @@ public static String encodeObjectToString(T t) { } } - public static Map getPartitionInfoMap(Table table, BinaryRow partitionValues) { + public static Map getPartitionInfoMap(Table table, BinaryRow partitionValues, String timeZone) { Map partitionInfoMap = new HashMap<>(); List partitionKeys = table.partitionKeys(); RowType partitionType = table.rowType().project(partitionKeys); @@ -398,7 +398,7 @@ public static Map getPartitionInfoMap(Table table, BinaryRow par for (int i = 0; i < partitionKeys.size(); i++) { try { String partitionValue = serializePartitionValue(partitionType.getFields().get(i).type(), - partitionValuesArray[i]); + partitionValuesArray[i], timeZone); partitionInfoMap.put(partitionKeys.get(i), partitionValue); } catch (UnsupportedOperationException e) { LOG.warn("Failed to serialize partition value for key {}: {}", partitionKeys.get(i), e.getMessage()); @@ -408,7 +408,8 @@ public static Map getPartitionInfoMap(Table table, BinaryRow par return partitionInfoMap; } - private static String serializePartitionValue(org.apache.paimon.types.DataType type, Object value) { + private static String serializePartitionValue(org.apache.paimon.types.DataType type, Object value, + String timeZone) { if (value == null) { return "\\N"; } @@ -432,16 +433,19 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME_WITHOUT_TIME_ZONE: - // Paimon time is stored as microseconds since midnight + // Paimon time is stored as microseconds since midnight in utc long micros = (Long) value; LocalDateTime time = LocalDateTime.ofEpochSecond(micros / 1_000_000, (int) (micros % 1_000_000) * 1000, - ZoneId.systemDefault().getRules().getOffset(Instant.now())); + ZoneId.of("UTC").getRules().getOffset(Instant.now())); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP_WITHOUT_TIME_ZONE: + // Paimon timestamp is stored as Timestamp type in utc + return ((Timestamp) value).toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + // Paimon timestamp with local time zone is stored as Timestamp type in utc Timestamp timestamp = (Timestamp) value; return timestamp.toLocalDateTime() - .atZone(ZoneId.systemDefault()) + .atZone(ZoneId.of(timeZone)) // Convert to local time zone .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index eefbfc064df4d2..91b57e1908543d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -276,8 +276,11 @@ public List getSplits(int numBackends) throws UserException { BinaryRow partitionValue = dataSplit.partition(); selectedPartitionValues.add(partitionValue); - Map partitionInfoMap = PaimonUtil.getPartitionInfoMap( - source.getPaimonTable(), partitionValue); + // try to get partition info map if EnableRuntimeFilterPartitionPrune is true + Map partitionInfoMap = sessionVariable.isEnableRuntimeFilterPartitionPrune() + ? PaimonUtil.getPartitionInfoMap( + source.getPaimonTable(), partitionValue, sessionVariable.getTimeZone()) + : null; Optional> optRawFiles = dataSplit.convertToRawFiles(); Optional> optDeletionFiles = dataSplit.deletionFiles(); if (applyCountPushdown && dataSplit.mergedRowCountAvailable()) { From 460c6475337476c7022b5e9d825e515aa79f4d72 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 15:29:00 +0800 Subject: [PATCH 19/43] fix time --- .../datasource/iceberg/IcebergUtils.java | 25 +++++++++++++------ .../doris/datasource/paimon/PaimonUtil.java | 4 +-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index f49a5fe1c3ddb2..dfe8325339ed70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -116,10 +116,12 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.Month; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Base64; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -637,22 +639,29 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type case UUID: case DECIMAL: return value.toString(); + case FIXED: + case BINARY: + // Fixed and binary types are stored as ByteBuffer + ByteBuffer buffer = (ByteBuffer) value; + byte[] res = new byte[buffer.limit()]; + buffer.get(res); + return Base64.getEncoder().encodeToString(res); case DATE: - // Iceberg date is stored as days since epoch + // Iceberg date is stored as days since epoch (1970-01-01) LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME: - // Iceberg time is stored as microseconds since midnight without timezone + // Iceberg time is stored as microseconds since midnight long micros = (Long) value; - LocalDateTime time = LocalDateTime.ofEpochSecond(micros / 1_000_000, (int) (micros % 1_000_000) * 1000, - ZoneId.of("UTC").getRules().getOffset(Instant.now())); + LocalTime time = LocalTime.ofNanoOfDay(micros * 1000); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP: - // Iceberg timestamp is stored as microseconds since epoch without timezone + // Iceberg timestamp is stored as microseconds since epoch + // (1970-01-01T00:00:00Z) long timestampMicros = (Long) value; - LocalDateTime timestamp = LocalDateTime.ofInstant( - Instant.ofEpochSecond(timestampMicros / 1_000_000, (int) (timestampMicros % 1_000_000) * 1000), - ZoneId.of("UTC")); + LocalDateTime timestamp = LocalDateTime.ofEpochSecond( + timestampMicros / 1_000_000, (int) (timestampMicros % 1_000_000) * 1000, + ZoneId.of("UTC").getRules().getOffset(Instant.now())); return timestamp.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 330d737116c4f5..7170a5eab5c119 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -71,6 +71,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -435,8 +436,7 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t case TIME_WITHOUT_TIME_ZONE: // Paimon time is stored as microseconds since midnight in utc long micros = (Long) value; - LocalDateTime time = LocalDateTime.ofEpochSecond(micros / 1_000_000, (int) (micros % 1_000_000) * 1000, - ZoneId.of("UTC").getRules().getOffset(Instant.now())); + LocalTime time = LocalTime.ofNanoOfDay(micros * 1000); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP_WITHOUT_TIME_ZONE: // Paimon timestamp is stored as Timestamp type in utc From 61b5a0a4d7ae7e72b6a9c3f9656577dd0c99fde6 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 15:39:15 +0800 Subject: [PATCH 20/43] fix case --- .../iceberg/run18.sql | 305 ++++++++++++++---- .../paimon/run06.sql | 229 +++++++++---- 2 files changed, 419 insertions(+), 115 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql index cd02c106610ad4..3587c9c8e85bb4 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql @@ -1,45 +1,236 @@ create database if not exists demo.partition_db; + use demo.partition_db; +-- set time zone to shanghai +set spark.sql.session.timeZone = 'Asia/Shanghai'; -- Partition by date type CREATE TABLE date_partitioned ( id BIGINT, name STRING, partition_key DATE -) USING ICEBERG -PARTITIONED BY (partition_key); +) USING ICEBERG PARTITIONED BY (partition_key); +-- Insert data into date_partitioned table +INSERT INTO + date_partitioned +VALUES (1, 'Alice', DATE '2024-01-01'), + (2, 'Bob', DATE '2024-01-01'), + ( + 3, + 'Charlie', + DATE '2024-02-01' + ), + (4, 'David', DATE '2024-02-01'), + (5, 'Eve', DATE '2024-03-01'); + +-- Partition by time type +CREATE TABLE time_partitioned ( + id BIGINT, + name STRING, + partition_key TIME +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into time_partitioned table +INSERT INTO + time_partitioned +VALUES ( + 1, + 'Morning Shift', + TIME '08:00:00' + ), + ( + 2, + 'Afternoon Shift', + TIME '12:00:00' + ), + ( + 3, + 'Evening Shift', + TIME '18:00:00' + ), + ( + 4, + 'Night Shift', + TIME '22:00:00' + ), + ( + 5, + 'Overnight Shift', + TIME '02:00:00' + ); -- Partition by integer type CREATE TABLE int_partitioned ( id BIGINT, name STRING, partition_key INT -) USING ICEBERG -PARTITIONED BY (partition_key); +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into int_partitioned table +INSERT INTO + int_partitioned +VALUES (1, 'Product A', 1), + (2, 'Product B', 1), + (3, 'Product C', 2), + (4, 'Product D', 2), + (5, 'Product E', 3); + +-- Partition by float type +CREATE TABLE float_partitioned ( + id BIGINT, + name STRING, + partition_key FLOAT +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into float_partitioned table +INSERT INTO + float_partitioned +VALUES (1, 'Item 1', 10.5), + (2, 'Item 2', 20.75), + (3, 'Item 3', 30.0), + (4, 'Item 4', 40.25), + (5, 'Item 5', 50.5); -- Partition by string type CREATE TABLE string_partitioned ( id BIGINT, name STRING, partition_key STRING -) USING ICEBERG -PARTITIONED BY (partition_key); +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into string_partitioned table +INSERT INTO + string_partitioned +VALUES (1, 'User1', 'North America'), + (2, 'User2', 'North America'), + (3, 'User3', 'Europe'), + (4, 'User4', 'Europe'), + (5, 'User5', 'Asia'), + (6, 'User6', 'Asia'); + +-- Partition by fixed type +CREATE TABLE fixed_partitioned ( + id BIGINT, + name STRING, + partition_key FIXED(16) +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into fixed_partitioned table +INSERT INTO + fixed_partitioned +VALUES ( + 1, + 'Product A', + CAST(1 AS FIXED(16)) + ), + ( + 2, + 'Product B', + CAST(1 AS FIXED(16)) + ), + ( + 3, + 'Product C', + CAST(2 AS FIXED(16)) + ), + ( + 4, + 'Product D', + CAST(2 AS FIXED(16)) + ), + ( + 5, + 'Product E', + CAST(3 AS FIXED(16)) + ); + +-- Partition by uuid type +CREATE TABLE uuid_partitioned ( + id BIGINT, + name STRING, + partition_key UUID +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into uuid_partitioned table +INSERT INTO + uuid_partitioned +VALUES ( + 1, + 'UUID User 1', + UUID '123e4567-e89b-12d3-a456-426614174000' + ), + ( + 2, + 'UUID User 2', + UUID '123e4567-e89b-12d3-a456-426614174001' + ), + ( + 3, + 'UUID User 3', + UUID '123e4567-e89b-12d3-a456-426614174002' + ), + ( + 4, + 'UUID User 4', + UUID '123e4567-e89b-12d3-a456-426614174003' + ), + ( + 5, + 'UUID User 5', + UUID '123e4567-e89b-12d3-a456-426614174004' + ); -- Partition by timestamp type CREATE TABLE timestamp_partitioned ( id BIGINT, name STRING, partition_key TIMESTAMP -) USING ICEBERG -PARTITIONED BY (partition_key); +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into timestamp_partitioned table +INSERT INTO + timestamp_partitioned +VALUES ( + 1, + 'Event1', + TIMESTAMP '2024-01-15 08:00:00' + ), + ( + 2, + 'Event2', + TIMESTAMP '2024-01-15 09:00:00' + ), + ( + 3, + 'Event3', + TIMESTAMP '2024-01-15 14:00:00' + ), + ( + 4, + 'Event4', + TIMESTAMP '2024-01-16 10:00:00' + ), + ( + 5, + 'Event5', + TIMESTAMP '2024-01-16 16:00:00' + ); -- Partition by boolean type CREATE TABLE boolean_partitioned ( id BIGINT, name STRING, partition_key BOOLEAN -) USING ICEBERG -PARTITIONED BY (partition_key); +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into boolean_partitioned table +INSERT INTO + boolean_partitioned +VALUES (1, 'Active User', true), + (2, 'Active Admin', true), + (3, 'Inactive User', false), + (4, 'Inactive Guest', false), + (5, 'Active Manager', true); -- Partition by decimal type CREATE TABLE decimal_partitioned ( @@ -47,55 +238,49 @@ CREATE TABLE decimal_partitioned ( name STRING, value FLOAT, partition_key DECIMAL(10, 2) -) USING ICEBERG -PARTITIONED BY (partition_key); - --- Insert data into date_partitioned table -INSERT INTO date_partitioned VALUES -(1, 'Alice', DATE '2024-01-01'), -(2, 'Bob', DATE '2024-01-01'), -(3, 'Charlie', DATE '2024-02-01'), -(4, 'David', DATE '2024-02-01'), -(5, 'Eve', DATE '2024-03-01'); - --- Insert data into int_partitioned table -INSERT INTO int_partitioned VALUES -(1, 'Product A', 1), -(2, 'Product B', 1), -(3, 'Product C', 2), -(4, 'Product D', 2), -(5, 'Product E', 3); - --- Insert data into string_partitioned table -INSERT INTO string_partitioned VALUES -(1, 'User1', 'North America'), -(2, 'User2', 'North America'), -(3, 'User3', 'Europe'), -(4, 'User4', 'Europe'), -(5, 'User5', 'Asia'), -(6, 'User6', 'Asia'); - --- Insert data into timestamp_partitioned table -INSERT INTO timestamp_partitioned VALUES -(1, 'Event1', TIMESTAMP '2024-01-15 08:00:00'), -(2, 'Event2', TIMESTAMP '2024-01-15 09:00:00'), -(3, 'Event3', TIMESTAMP '2024-01-15 14:00:00'), -(4, 'Event4', TIMESTAMP '2024-01-16 10:00:00'), -(5, 'Event5', TIMESTAMP '2024-01-16 16:00:00'); - --- Insert data into boolean_partitioned table -INSERT INTO boolean_partitioned VALUES -(1, 'Active User', true), -(2, 'Active Admin', true), -(3, 'Inactive User', false), -(4, 'Inactive Guest', false), -(5, 'Active Manager', true); +) USING ICEBERG PARTITIONED BY (partition_key); -- Insert data into decimal_partitioned table -INSERT INTO decimal_partitioned VALUES -(1, 'Item A', 125.50, 10.50), -(2, 'Item B', 200.75, 10.50), -(3, 'Item C', 89.99, 25.25), -(4, 'Item D', 156.80, 25.25), -(5, 'Item E', 299.95, 50.00), -(6, 'Item F', 399.99, 50.00); \ No newline at end of file +INSERT INTO + decimal_partitioned +VALUES (1, 'Item A', 125.50, 10.50), + (2, 'Item B', 200.75, 10.50), + (3, 'Item C', 89.99, 25.25), + (4, 'Item D', 156.80, 25.25), + (5, 'Item E', 299.95, 50.00), + (6, 'Item F', 399.99, 50.00); +-- Partition by binary type +CREATE TABLE binary_partitioned ( + id BIGINT, + name STRING, + partition_key BINARY +) USING ICEBERG PARTITIONED BY (partition_key); + +-- Insert data into binary_partitioned table +INSERT INTO + binary_partitioned +VALUES ( + 1, + 'Binary Data 1', + CAST('01010101' AS BINARY) + ), + ( + 2, + 'Binary Data 2', + CAST('01010101' AS BINARY) + ), + ( + 3, + 'Binary Data 3', + CAST('11001100' AS BINARY) + ), + ( + 4, + 'Binary Data 4', + CAST('11001100' AS BINARY) + ), + ( + 5, + 'Binary Data 5', + CAST('11110000' AS BINARY) + ); \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql index c502a443b8323e..6dc5c0eed1f45d 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -4,6 +4,9 @@ create database if not exists partition_db; use partition_db; +-- set time zone to shanghai +set spark.sql.session.timeZone = 'Asia/Shanghai'; + -- Partition by date type CREATE TABLE date_partitioned ( id BIGINT, @@ -11,6 +14,90 @@ CREATE TABLE date_partitioned ( partition_key DATE ) PARTITIONED BY (partition_key); +-- Insert data into date_partitioned table +INSERT INTO + date_partitioned +VALUES (1, 'Alice', DATE '2024-01-01'), + (2, 'Bob', DATE '2024-01-01'), + ( + 3, + 'Charlie', + DATE '2024-02-01' + ), + (4, 'David', DATE '2024-02-01'), + (5, 'Eve', DATE '2024-03-01'); + +-- Partition by time type +CREATE TABLE time_partitioned ( + id BIGINT, + name STRING, + partition_key TIME +) PARTITIONED BY (partition_key); +-- Insert data into time_partitioned table +INSERT INTO + time_partitioned +VALUES ( + 1, + 'Morning Shift', + TIME '08:00:00' + ), + ( + 2, + 'Afternoon Shift', + TIME '12:00:00' + ), + ( + 3, + 'Evening Shift', + TIME '18:00:00' + ), + ( + 4, + 'Night Shift', + TIME '22:00:00' + ), + ( + 5, + 'Overnight Shift', + TIME '02:00:00' + ); + +-- Partition by timestamp type +CREATE TABLE timestamp_partitioned ( + id BIGINT, + name STRING, + partition_key TIMESTAMP +) PARTITIONED BY (partition_key); + +-- Insert data into timestamp_partitioned table +INSERT INTO + timestamp_partitioned +VALUES ( + 1, + 'Event1', + TIMESTAMP '2024-01-15 08:00:00' + ), + ( + 2, + 'Event2', + TIMESTAMP '2024-01-15 09:00:00' + ), + ( + 3, + 'Event3', + TIMESTAMP '2024-01-15 14:00:00' + ), + ( + 4, + 'Event4', + TIMESTAMP '2024-01-16 10:00:00' + ), + ( + 5, + 'Event5', + TIMESTAMP '2024-01-16 16:00:00' + ); + -- Partition by integer type CREATE TABLE int_partitioned ( id BIGINT, @@ -18,20 +105,48 @@ CREATE TABLE int_partitioned ( partition_key INT ) PARTITIONED BY (partition_key); --- Partition by string type -CREATE TABLE string_partitioned ( +-- Insert data into int_partitioned table +INSERT INTO + int_partitioned +VALUES (1, 'Product A', 1), + (2, 'Product B', 1), + (3, 'Product C', 2), + (4, 'Product D', 2), + (5, 'Product E', 3); + +-- Partition by bigint type +CREATE TABLE bigint_partitioned ( id BIGINT, name STRING, - partition_key STRING + partition_key BIGINT ) PARTITIONED BY (partition_key); --- Partition by timestamp type -CREATE TABLE timestamp_partitioned ( +-- Insert data into bigint_partitioned table +INSERT INTO + bigint_partitioned +VALUES (1, 'Item 1', 100), + (2, 'Item 2', 100), + (3, 'Item 3', 200), + (4, 'Item 4', 200), + (5, 'Item 5', 300); + +-- Partition by string type +CREATE TABLE string_partitioned ( id BIGINT, name STRING, - partition_key TIMESTAMP + partition_key STRING ) PARTITIONED BY (partition_key); +-- Insert data into string_partitioned table +INSERT INTO + string_partitioned +VALUES (1, 'User1', 'North America'), + (2, 'User2', 'North America'), + (3, 'User3', 'Europe'), + (4, 'User4', 'Europe'), + (5, 'User5', 'Asia'), + (6, 'User6', 'Asia'); + -- Partition by boolean type CREATE TABLE boolean_partitioned ( id BIGINT, @@ -39,61 +154,65 @@ CREATE TABLE boolean_partitioned ( partition_key BOOLEAN ) PARTITIONED BY (partition_key); +-- Insert data into boolean_partitioned table +INSERT INTO + boolean_partitioned +VALUES (1, 'Active User', true), + (2, 'Active Admin', true), + (3, 'Inactive User', false), + (4, 'Inactive Guest', false), + (5, 'Active Manager', true); + -- Partition by decimal type CREATE TABLE decimal_partitioned ( id BIGINT, name STRING, - value FLOAT, + value DOUBLE, partition_key DECIMAL(10, 2) ) PARTITIONED BY (partition_key); --- Insert data into date_partitioned table -INSERT INTO date_partitioned VALUES -(1, 'Alice', DATE '2024-01-01'), -(2, 'Bob', DATE '2024-01-01'), -(3, 'Charlie', DATE '2024-02-01'), -(4, 'David', DATE '2024-02-01'), -(5, 'Eve', DATE '2024-03-01'); - --- Insert data into int_partitioned table -INSERT INTO int_partitioned VALUES -(1, 'Product A', 1), -(2, 'Product B', 1), -(3, 'Product C', 2), -(4, 'Product D', 2), -(5, 'Product E', 3); - --- Insert data into string_partitioned table -INSERT INTO string_partitioned VALUES -(1, 'User1', 'North America'), -(2, 'User2', 'North America'), -(3, 'User3', 'Europe'), -(4, 'User4', 'Europe'), -(5, 'User5', 'Asia'), -(6, 'User6', 'Asia'); - --- Insert data into timestamp_partitioned table -INSERT INTO timestamp_partitioned VALUES -(1, 'Event1', TIMESTAMP '2024-01-15 08:00:00'), -(2, 'Event2', TIMESTAMP '2024-01-15 09:00:00'), -(3, 'Event3', TIMESTAMP '2024-01-15 14:00:00'), -(4, 'Event4', TIMESTAMP '2024-01-16 10:00:00'), -(5, 'Event5', TIMESTAMP '2024-01-16 16:00:00'); - --- Insert data into boolean_partitioned table -INSERT INTO boolean_partitioned VALUES -(1, 'Active User', true), -(2, 'Active Admin', true), -(3, 'Inactive User', false), -(4, 'Inactive Guest', false), -(5, 'Active Manager', true); - -- Insert data into decimal_partitioned table -INSERT INTO decimal_partitioned VALUES -(1, 'Item A', 125.50, 10.50), -(2, 'Item B', 200.75, 10.50), -(3, 'Item C', 89.99, 25.25), -(4, 'Item D', 156.80, 25.25), -(5, 'Item E', 299.95, 50.00), -(6, 'Item F', 399.99, 50.00); +INSERT INTO + decimal_partitioned +VALUES (1, 'Item A', 125.50, 10.50), + (2, 'Item B', 200.75, 10.50), + (3, 'Item C', 89.99, 25.25), + (4, 'Item D', 156.80, 25.25), + (5, 'Item E', 299.95, 50.00), + (6, 'Item F', 399.99, 50.00); + +-- Partition by binary type +CREATE TABLE binary_partitioned ( + id BIGINT, + name STRING, + partition_key BINARY +) PARTITIONED BY (partition_key); +-- Insert data into binary_partitioned table +INSERT INTO + binary_partitioned +VALUES ( + 1, + 'Binary Data 1', + CAST('binary1' AS BINARY) + ), + ( + 2, + 'Binary Data 2', + CAST('binary1' AS BINARY) + ), + ( + 3, + 'Binary Data 3', + CAST('binary2' AS BINARY) + ), + ( + 4, + 'Binary Data 4', + CAST('binary2' AS BINARY) + ), + ( + 5, + 'Binary Data 5', + CAST('binary3' AS BINARY) + ); \ No newline at end of file From be392210c212149b742bba92699d9ec0c1e558e5 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 21:39:16 +0800 Subject: [PATCH 21/43] flush paimon case --- .../iceberg/run18.sql | 117 ++++-------------- .../paimon/run06.sql | 54 +++----- .../doris/datasource/paimon/PaimonUtil.java | 6 +- ...aimon_runtime_filter_partition_pruning.out | 52 +++++++- ...on_runtime_filter_partition_pruning.groovy | 50 ++++++++ 5 files changed, 146 insertions(+), 133 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql index 3587c9c8e85bb4..2e3f9ac08505c4 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql @@ -3,7 +3,7 @@ create database if not exists demo.partition_db; use demo.partition_db; -- set time zone to shanghai -set spark.sql.session.timeZone = 'Asia/Shanghai'; +SET TIME ZONE '+08:00'; -- Partition by date type CREATE TABLE date_partitioned ( id BIGINT, @@ -30,35 +30,6 @@ CREATE TABLE time_partitioned ( partition_key TIME ) USING ICEBERG PARTITIONED BY (partition_key); --- Insert data into time_partitioned table -INSERT INTO - time_partitioned -VALUES ( - 1, - 'Morning Shift', - TIME '08:00:00' - ), - ( - 2, - 'Afternoon Shift', - TIME '12:00:00' - ), - ( - 3, - 'Evening Shift', - TIME '18:00:00' - ), - ( - 4, - 'Night Shift', - TIME '22:00:00' - ), - ( - 5, - 'Overnight Shift', - TIME '02:00:00' - ); - -- Partition by integer type CREATE TABLE int_partitioned ( id BIGINT, @@ -108,112 +79,76 @@ VALUES (1, 'User1', 'North America'), (5, 'User5', 'Asia'), (6, 'User6', 'Asia'); --- Partition by fixed type -CREATE TABLE fixed_partitioned ( - id BIGINT, - name STRING, - partition_key FIXED(16) -) USING ICEBERG PARTITIONED BY (partition_key); - --- Insert data into fixed_partitioned table -INSERT INTO - fixed_partitioned -VALUES ( - 1, - 'Product A', - CAST(1 AS FIXED(16)) - ), - ( - 2, - 'Product B', - CAST(1 AS FIXED(16)) - ), - ( - 3, - 'Product C', - CAST(2 AS FIXED(16)) - ), - ( - 4, - 'Product D', - CAST(2 AS FIXED(16)) - ), - ( - 5, - 'Product E', - CAST(3 AS FIXED(16)) - ); - --- Partition by uuid type -CREATE TABLE uuid_partitioned ( +-- Partition by timestamp type +CREATE TABLE timestamp_partitioned ( id BIGINT, name STRING, - partition_key UUID + partition_key TIMESTAMP ) USING ICEBERG PARTITIONED BY (partition_key); --- Insert data into uuid_partitioned table +-- Insert data into timestamp_partitioned table INSERT INTO - uuid_partitioned + timestamp_partitioned VALUES ( 1, - 'UUID User 1', - UUID '123e4567-e89b-12d3-a456-426614174000' + 'Event1', + TIMESTAMP '2024-01-15 08:00:00' ), ( 2, - 'UUID User 2', - UUID '123e4567-e89b-12d3-a456-426614174001' + 'Event2', + TIMESTAMP '2024-01-15 09:00:00' ), ( 3, - 'UUID User 3', - UUID '123e4567-e89b-12d3-a456-426614174002' + 'Event3', + TIMESTAMP '2024-01-15 14:00:00' ), ( 4, - 'UUID User 4', - UUID '123e4567-e89b-12d3-a456-426614174003' + 'Event4', + TIMESTAMP '2024-01-16 10:00:00' ), ( 5, - 'UUID User 5', - UUID '123e4567-e89b-12d3-a456-426614174004' + 'Event5', + TIMESTAMP '2024-01-16 16:00:00' ); --- Partition by timestamp type -CREATE TABLE timestamp_partitioned ( +-- Partition by timestamp_ntz type +CREATE TABLE timestamp_ntz_partitioned ( id BIGINT, name STRING, - partition_key TIMESTAMP + partition_key TIMESTAMP_NTZ ) USING ICEBERG PARTITIONED BY (partition_key); --- Insert data into timestamp_partitioned table +-- INSERT INTO timestamp_ntz_partitioned INSERT INTO - timestamp_partitioned + timestamp_ntz_partitioned VALUES ( 1, 'Event1', - TIMESTAMP '2024-01-15 08:00:00' + TIMESTAMP_NTZ '2024-01-15 08:00:00' ), ( 2, 'Event2', - TIMESTAMP '2024-01-15 09:00:00' + TIMESTAMP_NTZ '2024-01-15 09:00:00' ), ( 3, 'Event3', - TIMESTAMP '2024-01-15 14:00:00' + TIMESTAMP_NTZ '2024-01-15 14:00:00' ), ( 4, 'Event4', - TIMESTAMP '2024-01-16 10:00:00' + TIMESTAMP_NTZ '2024-01-16 10:00:00' ), ( 5, 'Event5', - TIMESTAMP '2024-01-16 16:00:00' + TIMESTAMP_NTZ '2024-01-16 16:00:00' ); -- Partition by boolean type diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql index 6dc5c0eed1f45d..bcadfd7be3564b 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -5,7 +5,7 @@ create database if not exists partition_db; use partition_db; -- set time zone to shanghai -set spark.sql.session.timeZone = 'Asia/Shanghai'; +SET TIME ZONE '+08:00'; -- Partition by date type CREATE TABLE date_partitioned ( @@ -27,41 +27,6 @@ VALUES (1, 'Alice', DATE '2024-01-01'), (4, 'David', DATE '2024-02-01'), (5, 'Eve', DATE '2024-03-01'); --- Partition by time type -CREATE TABLE time_partitioned ( - id BIGINT, - name STRING, - partition_key TIME -) PARTITIONED BY (partition_key); --- Insert data into time_partitioned table -INSERT INTO - time_partitioned -VALUES ( - 1, - 'Morning Shift', - TIME '08:00:00' - ), - ( - 2, - 'Afternoon Shift', - TIME '12:00:00' - ), - ( - 3, - 'Evening Shift', - TIME '18:00:00' - ), - ( - 4, - 'Night Shift', - TIME '22:00:00' - ), - ( - 5, - 'Overnight Shift', - TIME '02:00:00' - ); - -- Partition by timestamp type CREATE TABLE timestamp_partitioned ( id BIGINT, @@ -215,4 +180,19 @@ VALUES ( 5, 'Binary Data 5', CAST('binary3' AS BINARY) - ); \ No newline at end of file + ); + +-- Partition by float type +CREATE TABLE float_partitioned ( + id BIGINT, + name STRING, + partition_key FLOAT +) PARTITIONED BY (partition_key); +-- Insert data into float_partitioned table +INSERT INTO + float_partitioned +VALUES (1, 'Float Data 1', 1.5), + (2, 'Float Data 2', 1.5), + (3, 'Float Data 3', 2.5), + (4, 'Float Data 4', 2.5), + (5, 'Float Data 5', 3.5); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 7170a5eab5c119..ece9a5166e3b3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -68,9 +68,7 @@ import org.apache.paimon.utils.RowDataToObjectArrayConverter; import java.io.IOException; -import java.time.Instant; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -445,7 +443,9 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t // Paimon timestamp with local time zone is stored as Timestamp type in utc Timestamp timestamp = (Timestamp) value; return timestamp.toLocalDateTime() - .atZone(ZoneId.of(timeZone)) // Convert to local time zone + .atZone(ZoneId.of("UTC")) + .withZoneSameInstant(ZoneId.of(timeZone)) + .toLocalDateTime() .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index 093cac83259e0e..17e942c2f8d9e0 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -35,12 +35,36 @@ -- !runtime_filter_partition_pruning_timestamp2 -- 2 +-- !runtime_filter_partition_pruning_bigint1 -- +1 + +-- !runtime_filter_partition_pruning_bigint2 -- +3 + +-- !runtime_filter_partition_pruning_bigint3 -- +1 + -- !runtime_filter_partition_pruning_boolean1 -- 3 -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_binary1 -- +0 + +-- !runtime_filter_partition_pruning_binary2 -- +0 + +-- !runtime_filter_partition_pruning_float1 -- +1 + +-- !runtime_filter_partition_pruning_float2 -- +3 + +-- !runtime_filter_partition_pruning_float3 -- +1 + -- !runtime_filter_partition_pruning_decimal1 -- 2 @@ -72,10 +96,19 @@ 3 -- !runtime_filter_partition_pruning_timestamp1 -- -0 +1 -- !runtime_filter_partition_pruning_timestamp2 -- -0 +2 + +-- !runtime_filter_partition_pruning_bigint1 -- +1 + +-- !runtime_filter_partition_pruning_bigint2 -- +3 + +-- !runtime_filter_partition_pruning_bigint3 -- +1 -- !runtime_filter_partition_pruning_boolean1 -- 3 @@ -83,3 +116,18 @@ -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_binary1 -- +0 + +-- !runtime_filter_partition_pruning_binary2 -- +0 + +-- !runtime_filter_partition_pruning_float1 -- +1 + +-- !runtime_filter_partition_pruning_float2 -- +3 + +-- !runtime_filter_partition_pruning_float3 -- +1 + diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy index e65c23b12456a8..bb3ba0afea076e 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy @@ -111,6 +111,24 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_bigint1 """ + select count(*) from bigint_partitioned where partition_key = + (select partition_key from bigint_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_bigint2 """ + select count(*) from bigint_partitioned where partition_key in + (select partition_key from bigint_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_bigint3 """ + select count(*) from bigint_partitioned where abs(partition_key) = + (select partition_key from bigint_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ qt_runtime_filter_partition_pruning_boolean1 """ select count(*) from boolean_partitioned where partition_key = (select partition_key from boolean_partitioned @@ -122,15 +140,47 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa (select partition_key from boolean_partitioned group by partition_key having count(*) > 0); """ + qt_runtime_filter_partition_pruning_binary1 """ + select count(*) from binary_partitioned where partition_key = + (select partition_key from binary_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_binary2 """ + select count(*) from binary_partitioned where partition_key in + (select partition_key from binary_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_float1 """ + select count(*) from float_partitioned where partition_key = + (select partition_key from float_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_float2 """ + select count(*) from float_partitioned where partition_key in + (select partition_key from float_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_float3 """ + select count(*) from float_partitioned where abs(partition_key) = + (select partition_key from float_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ } try { + sql """ set time_zone = 'Asia/Shanghai'; """ sql """ set enable_runtime_filter_partition_prune = false; """ test_runtime_filter_partition_pruning() sql """ set enable_runtime_filter_partition_prune = true; """ test_runtime_filter_partition_pruning() } finally { + sql """ unset variable time_zone; """ sql """ set enable_runtime_filter_partition_prune = true; """ } } From 3b6e49a8a9c3a0b1276689af7f8b430a7da56a80 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 21:48:52 +0800 Subject: [PATCH 22/43] flush iceberg cases --- .../iceberg/run18.sql | 7 --- ...eberg_runtime_filter_partition_pruning.out | 46 ++++++++++++++++++- ...rg_runtime_filter_partition_pruning.groovy | 44 ++++++++++++++++++ 3 files changed, 88 insertions(+), 9 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql index 2e3f9ac08505c4..23adcc82aa0d78 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql @@ -23,13 +23,6 @@ VALUES (1, 'Alice', DATE '2024-01-01'), (4, 'David', DATE '2024-02-01'), (5, 'Eve', DATE '2024-03-01'); --- Partition by time type -CREATE TABLE time_partitioned ( - id BIGINT, - name STRING, - partition_key TIME -) USING ICEBERG PARTITIONED BY (partition_key); - -- Partition by integer type CREATE TABLE int_partitioned ( id BIGINT, diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index 16dc58415ece1c..2847e2debdcc3e 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -41,6 +41,27 @@ -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_float1 -- +1 + +-- !runtime_filter_partition_pruning_float2 -- +2 + +-- !runtime_filter_partition_pruning_float3 -- +1 + +-- !runtime_filter_partition_pruning_timestamp_ntz1 -- +1 + +-- !runtime_filter_partition_pruning_timestamp_ntz2 -- +2 + +-- !runtime_filter_partition_pruning_binary1 -- +1 + +-- !runtime_filter_partition_pruning_binary2 -- +3 + -- !runtime_filter_partition_pruning_decimal1 -- 2 @@ -72,10 +93,10 @@ 3 -- !runtime_filter_partition_pruning_timestamp1 -- -1 +0 -- !runtime_filter_partition_pruning_timestamp2 -- -2 +0 -- !runtime_filter_partition_pruning_boolean1 -- 3 @@ -83,3 +104,24 @@ -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_float1 -- +1 + +-- !runtime_filter_partition_pruning_float2 -- +2 + +-- !runtime_filter_partition_pruning_float3 -- +1 + +-- !runtime_filter_partition_pruning_timestamp_ntz1 -- +1 + +-- !runtime_filter_partition_pruning_timestamp_ntz2 -- +2 + +-- !runtime_filter_partition_pruning_binary1 -- +0 + +-- !runtime_filter_partition_pruning_binary2 -- +0 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index c8344204c68153..630a95f267fe68 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -127,14 +127,58 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern (select partition_key from boolean_partitioned group by partition_key having count(*) > 0); """ + qt_runtime_filter_partition_pruning_float1 """ + select count(*) from float_partitioned where partition_key = + (select partition_key from float_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_float2 """ + select count(*) from float_partitioned where partition_key in + (select partition_key from float_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_float3 """ + select count(*) from float_partitioned where abs(partition_key) = + (select partition_key from float_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_timestamp_ntz1 """ + select count(*) from timestamp_ntz_partitioned where partition_key = + (select partition_key from timestamp_ntz_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_timestamp_ntz2 """ + select count(*) from timestamp_ntz_partitioned where partition_key in + (select partition_key from timestamp_ntz_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ + qt_runtime_filter_partition_pruning_binary1 """ + select count(*) from binary_partitioned where partition_key = + (select partition_key from binary_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 1); + """ + qt_runtime_filter_partition_pruning_binary2 """ + select count(*) from binary_partitioned where partition_key in + (select partition_key from binary_partitioned + group by partition_key having count(*) > 0 + order by partition_key desc limit 2); + """ } try { + sql """ set time_zone = 'Asia/Shanghai'; """ sql """ set enable_runtime_filter_partition_prune = false; """ test_runtime_filter_partition_pruning() sql """ set enable_runtime_filter_partition_prune = true; """ test_runtime_filter_partition_pruning() } finally { + sql """ unset variable time_zone; """ sql """ set enable_runtime_filter_partition_prune = true; """ } From ecf4c9cba042d35654ecd6c769a98947e7d157af Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 22:37:56 +0800 Subject: [PATCH 23/43] fix iceberg timestamp with time zone --- .../datasource/iceberg/IcebergUtils.java | 19 +++++++++++++------ .../iceberg/source/IcebergScanNode.java | 3 ++- ...eberg_runtime_filter_partition_pruning.out | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index dfe8325339ed70..da70ec81679bb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -102,6 +102,7 @@ import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; +import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.util.LocationUtil; import org.apache.iceberg.util.SnapshotUtil; import org.apache.iceberg.util.StructProjection; @@ -119,6 +120,7 @@ import java.time.LocalTime; import java.time.Month; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Base64; @@ -607,14 +609,14 @@ public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type) { } } - public static Map getPartitionInfoMap(PartitionData partitionData) { + public static Map getPartitionInfoMap(PartitionData partitionData, String timeZone) { Map partitionInfoMap = new HashMap<>(); List fields = partitionData.getPartitionType().asNestedType().fields(); for (int i = 0; i < fields.size(); i++) { NestedField field = fields.get(i); Object value = partitionData.get(i); try { - String partitionString = serializePartitionValue(field.type(), value); + String partitionString = serializePartitionValue(field.type(), value, timeZone); partitionInfoMap.put(field.name(), partitionString); } catch (UnsupportedOperationException e) { LOG.warn("Failed to serialize partition value for field {}: {}", field.name(), e.getMessage()); @@ -624,11 +626,10 @@ public static Map getPartitionInfoMap(PartitionData partitionDat return partitionInfoMap; } - private static String serializePartitionValue(org.apache.iceberg.types.Type type, Object value) { + private static String serializePartitionValue(org.apache.iceberg.types.Type type, Object value, String timeZone) { if (value == null) { return "\\N"; } - switch (type.typeId()) { case BOOLEAN: case INTEGER: @@ -657,11 +658,17 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP: // Iceberg timestamp is stored as microseconds since epoch - // (1970-01-01T00:00:00Z) + // (1970-01-01T00:00:00) long timestampMicros = (Long) value; + TimestampType timestampType = (TimestampType) type; LocalDateTime timestamp = LocalDateTime.ofEpochSecond( timestampMicros / 1_000_000, (int) (timestampMicros % 1_000_000) * 1000, - ZoneId.of("UTC").getRules().getOffset(Instant.now())); + ZoneOffset.UTC); + // type is timestamptz if timestampType.shouldAdjustToUTC() is true + if (timestampType.shouldAdjustToUTC()) { + timestamp = timestamp.atZone(ZoneId.of("UTC")).withZoneSameInstant(ZoneId.of(timeZone)) + .toLocalDateTime(); + } return timestamp.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); default: throw new UnsupportedOperationException("Unsupported type for serializePartitionValue: " + type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 14b507ac142410..4780d0c8feae4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -334,7 +334,8 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { if (isPartitionedTable) { PartitionData partitionData = (PartitionData) fileScanTask.file().partition(); if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { - split.setIcebergPartitionValues(IcebergUtils.getPartitionInfoMap(partitionData)); + split.setIcebergPartitionValues( + IcebergUtils.getPartitionInfoMap(partitionData, sessionVariable.getTimeZone())); } // Counts the number of partitions read partitionPathSet.add(partitionData.toString()); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index 2847e2debdcc3e..f78a263a07f2a3 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -93,10 +93,10 @@ 3 -- !runtime_filter_partition_pruning_timestamp1 -- -0 +1 -- !runtime_filter_partition_pruning_timestamp2 -- -0 +2 -- !runtime_filter_partition_pruning_boolean1 -- 3 From e7be73db0586c76f7c5ecc506d758451274e4342 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 23:22:18 +0800 Subject: [PATCH 24/43] fix binary as partition key --- be/src/vec/exec/scan/file_scanner.cpp | 3 +-- .../datasource/iceberg/IcebergUtils.java | 4 +-- .../doris/datasource/paimon/PaimonUtil.java | 3 ++- ...eberg_runtime_filter_partition_pruning.out | 4 +-- ...aimon_runtime_filter_partition_pruning.out | 12 --------- ...on_runtime_filter_partition_pruning.groovy | 25 ++++++++++--------- 6 files changed, 20 insertions(+), 31 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 67ef95ae1ef777..c234db96e02962 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -1493,8 +1493,7 @@ Status FileScanner::_generate_data_lake_partition_columns() { const auto& partition_values = _current_range.data_lake_partition_values; for (const auto& [key, value] : partition_values) { if (_all_col_name_to_slot_desc.find(key) == _all_col_name_to_slot_desc.end()) { - return Status::InternalError("Unknown data lake partition column, col_name={}", - key); + continue; // skip if the key is not in the slot desc map } const auto* slot_desc = _all_col_name_to_slot_desc[key]; _data_lake_partition_col_descs.emplace(key, std::make_tuple(value, slot_desc)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index da70ec81679bb6..a9e317c31633df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -113,6 +113,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.time.DateTimeException; import java.time.Instant; import java.time.LocalDate; @@ -123,7 +124,6 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Base64; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -646,7 +646,7 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type ByteBuffer buffer = (ByteBuffer) value; byte[] res = new byte[buffer.limit()]; buffer.get(res); - return Base64.getEncoder().encodeToString(res); + return new String(res, StandardCharsets.UTF_8); case DATE: // Iceberg date is stored as days since epoch (1970-01-01) LocalDate date = LocalDate.ofEpochDay((Integer) value); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index ece9a5166e3b3e..76ec6215bfe1ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -68,6 +68,7 @@ import org.apache.paimon.utils.RowDataToObjectArrayConverter; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneId; @@ -426,7 +427,7 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t return value.toString(); case BINARY: case VARBINARY: - return Base64.getEncoder().encodeToString((byte[]) value); + return new String((byte[]) value, StandardCharsets.UTF_8); case DATE: // Paimon date is stored as days since epoch LocalDate date = LocalDate.ofEpochDay((Integer) value); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index f78a263a07f2a3..d1e5dce9e57826 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -120,8 +120,8 @@ 2 -- !runtime_filter_partition_pruning_binary1 -- -0 +1 -- !runtime_filter_partition_pruning_binary2 -- -0 +3 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index 17e942c2f8d9e0..acd471479f862c 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -50,12 +50,6 @@ -- !runtime_filter_partition_pruning_boolean2 -- 5 --- !runtime_filter_partition_pruning_binary1 -- -0 - --- !runtime_filter_partition_pruning_binary2 -- -0 - -- !runtime_filter_partition_pruning_float1 -- 1 @@ -116,12 +110,6 @@ -- !runtime_filter_partition_pruning_boolean2 -- 5 --- !runtime_filter_partition_pruning_binary1 -- -0 - --- !runtime_filter_partition_pruning_binary2 -- -0 - -- !runtime_filter_partition_pruning_float1 -- 1 diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy index bb3ba0afea076e..3e99892cd92c83 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy @@ -140,18 +140,19 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa (select partition_key from boolean_partitioned group by partition_key having count(*) > 0); """ - qt_runtime_filter_partition_pruning_binary1 """ - select count(*) from binary_partitioned where partition_key = - (select partition_key from binary_partitioned - group by partition_key having count(*) > 0 - order by partition_key desc limit 1); - """ - qt_runtime_filter_partition_pruning_binary2 """ - select count(*) from binary_partitioned where partition_key in - (select partition_key from binary_partitioned - group by partition_key having count(*) > 0 - order by partition_key desc limit 2); - """ + // binary type as partition key will cause issues in paimon, so skipping these tests + // qt_runtime_filter_partition_pruning_binary1 """ + // select count(*) from binary_partitioned where partition_key = + // (select partition_key from binary_partitioned + // group by partition_key having count(*) > 0 + // order by partition_key desc limit 1); + // """ + // qt_runtime_filter_partition_pruning_binary2 """ + // select count(*) from binary_partitioned where partition_key in + // (select partition_key from binary_partitioned + // group by partition_key having count(*) > 0 + // order by partition_key desc limit 2); + // """ qt_runtime_filter_partition_pruning_float1 """ select count(*) from float_partitioned where partition_key = (select partition_key from float_partitioned From 02799287f4145a19df99cc9443f3bd986038b2ed Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 22 Jul 2025 23:56:00 +0800 Subject: [PATCH 25/43] fix hudi --- .../org/apache/doris/datasource/hudi/HudiUtils.java | 13 +++++++++++++ .../doris/datasource/hudi/source/HudiScanNode.java | 9 +++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java index b12b30e518e41a..fc1776aaa28bdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java @@ -32,6 +32,7 @@ import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; +import org.apache.doris.datasource.hive.HivePartition; import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; @@ -59,7 +60,9 @@ import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -402,4 +405,14 @@ public static TSchema getSchemaInfo(InternalSchema hudiInternalSchema) { return tschema; } + public static Map getHudiPartitionInfoMap(HMSExternalTable table, HivePartition partition) { + Map partitionInfoMap = new HashMap<>(); + List partitionColumns = table.getPartitionColumns(); + for (int i = 0; i < partitionColumns.size(); i++) { + String partitionName = partitionColumns.get(i).getName(); + String partitionValue = partition.getPartitionValues().get(i); + partitionInfoMap.put(partitionName, partitionValue); + } + return partitionInfoMap; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index ea48691b9dd450..d140208873c159 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -74,7 +74,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -377,10 +376,6 @@ private void getPartitionSplits(HivePartition partition, List splits) thr if (canUseNativeReader()) { - Map partitionValues = new HashMap<>(); - for (int i = 0; i < partition.getPartitionValues().size(); i++) { - partitionValues.put(partition.getColumns().get(i).getName(), partition.getPartitionValues().get(i)); - } fsView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { noLogsSplitNum.incrementAndGet(); String filePath = baseFile.getPath(); @@ -391,7 +386,9 @@ private void getPartitionSplits(HivePartition partition, List splits) thr HudiSplit hudiSplit = new HudiSplit(locationPath, 0, fileSize, fileSize, new String[0], partition.getPartitionValues()); hudiSplit.setTableFormatType(TableFormatType.HUDI); - hudiSplit.setHudiPartitionValues(partitionValues); + if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { + hudiSplit.setHudiPartitionValues(HudiUtils.getHudiPartitionInfoMap(hmsTable, partition)); + } splits.add(hudiSplit); }); } else { From 77fa6bc0f241d9ddf55b31a18d4e25482de465ce Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 23 Jul 2025 00:39:38 +0800 Subject: [PATCH 26/43] hudi p2 --- .../doris/datasource/hudi/HudiUtils.java | 2 +- .../datasource/hudi/source/HudiScanNode.java | 4 +- ..._hudi_runtime_filter_partition_pruning.out | 103 ++++++++++++++++++ ...di_runtime_filter_partition_pruning.groovy | 103 +++--------------- 4 files changed, 119 insertions(+), 93 deletions(-) create mode 100644 regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java index fc1776aaa28bdf..d153d62de7ac27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java @@ -405,7 +405,7 @@ public static TSchema getSchemaInfo(InternalSchema hudiInternalSchema) { return tschema; } - public static Map getHudiPartitionInfoMap(HMSExternalTable table, HivePartition partition) { + public static Map getPartitionInfoMap(HMSExternalTable table, HivePartition partition) { Map partitionInfoMap = new HashMap<>(); List partitionColumns = table.getPartitionColumns(); for (int i = 0; i < partitionColumns.size(); i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index d140208873c159..d79255bf421d0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -363,7 +363,6 @@ private List getIncrementalSplits() { incrementalRelation.getEndTs())).collect(Collectors.toList()); } - private void getPartitionSplits(HivePartition partition, List splits) throws IOException { String partitionName; @@ -374,7 +373,6 @@ private void getPartitionSplits(HivePartition partition, List splits) thr new StoragePath(partition.getPath())); } - if (canUseNativeReader()) { fsView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { noLogsSplitNum.incrementAndGet(); @@ -387,7 +385,7 @@ private void getPartitionSplits(HivePartition partition, List splits) thr new String[0], partition.getPartitionValues()); hudiSplit.setTableFormatType(TableFormatType.HUDI); if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { - hudiSplit.setHudiPartitionValues(HudiUtils.getHudiPartitionInfoMap(hmsTable, partition)); + hudiSplit.setHudiPartitionValues(HudiUtils.getPartitionInfoMap(hmsTable, partition)); } splits.add(hudiSplit); }); diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out new file mode 100644 index 00000000000000..0d6c5cd9e064da --- /dev/null +++ b/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out @@ -0,0 +1,103 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !runtime_filter_partition_pruning_boolean_1 -- +2 + +-- !runtime_filter_partition_pruning_boolean_2 -- +4 + +-- !runtime_filter_partition_pruning_tinyint_1 -- +2 + +-- !runtime_filter_partition_pruning_tinyint_2 -- +4 + +-- !runtime_filter_partition_pruning_smallint_1 -- +2 + +-- !runtime_filter_partition_pruning_smallint_2 -- +4 + +-- !runtime_filter_partition_pruning_int_1 -- +2 + +-- !runtime_filter_partition_pruning_int_2 -- +4 + +-- !runtime_filter_partition_pruning_int_3 -- +2 + +-- !runtime_filter_partition_pruning_bigint_1 -- +2 + +-- !runtime_filter_partition_pruning_bigint_2 -- +4 + +-- !runtime_filter_partition_pruning_string_1 -- +2 + +-- !runtime_filter_partition_pruning_string_2 -- +4 + +-- !runtime_filter_partition_pruning_date_1 -- +2 + +-- !runtime_filter_partition_pruning_date_2 -- +4 + +-- !runtime_filter_partition_pruning_complex_1 -- +7 + +-- !runtime_filter_partition_pruning_complex_2 -- +5 + +-- !runtime_filter_partition_pruning_boolean_1 -- +2 + +-- !runtime_filter_partition_pruning_boolean_2 -- +4 + +-- !runtime_filter_partition_pruning_tinyint_1 -- +2 + +-- !runtime_filter_partition_pruning_tinyint_2 -- +4 + +-- !runtime_filter_partition_pruning_smallint_1 -- +2 + +-- !runtime_filter_partition_pruning_smallint_2 -- +4 + +-- !runtime_filter_partition_pruning_int_1 -- +2 + +-- !runtime_filter_partition_pruning_int_2 -- +4 + +-- !runtime_filter_partition_pruning_int_3 -- +2 + +-- !runtime_filter_partition_pruning_bigint_1 -- +2 + +-- !runtime_filter_partition_pruning_bigint_2 -- +4 + +-- !runtime_filter_partition_pruning_string_1 -- +2 + +-- !runtime_filter_partition_pruning_string_2 -- +4 + +-- !runtime_filter_partition_pruning_date_1 -- +2 + +-- !runtime_filter_partition_pruning_date_2 -- +4 + +-- !runtime_filter_partition_pruning_complex_1 -- +7 + +-- !runtime_filter_partition_pruning_complex_2 -- +5 + diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy index 595dbf8b1f8ace..fb6e3e26bf0647 100644 --- a/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy @@ -22,7 +22,7 @@ suite("test_hudi_runtime_filter_partition_pruning", "p2,external,hudi,external_r return } - String catalog_name = "test_hudi_snapshot" + String catalog_name = "test_hudi_runtime_filter_partition_pruning" String props = context.config.otherConfigs.get("hudiEmrCatalog") sql """drop catalog if exists ${catalog_name};""" sql """ @@ -36,72 +36,6 @@ suite("test_hudi_runtime_filter_partition_pruning", "p2,external,hudi,external_r sql """ set enable_fallback_to_original_planner=false """ def test_runtime_filter_partition_pruning = { - // Test single partition table (INT type) - qt_runtime_filter_partition_pruning_one_partition_1 """ - select count(*) from one_partition_tb where part1 = - (select part1 from one_partition_tb - group by part1 having count(*) > 0 - order by part1 desc limit 1); - """ - - qt_runtime_filter_partition_pruning_one_partition_2 """ - select count(*) from one_partition_tb where part1 in - (select part1 from one_partition_tb - group by part1 having count(*) > 0 - order by part1 desc limit 2); - """ - - qt_runtime_filter_partition_pruning_one_partition_3 """ - select count(*) from one_partition_tb where abs(part1) = - (select part1 from one_partition_tb - group by part1 having count(*) > 0 - order by part1 desc limit 1); - """ - - // Test two partition table (STRING + INT types) - qt_runtime_filter_partition_pruning_two_partition_1 """ - select count(*) from two_partition_tb where part1 = - (select part1 from two_partition_tb - group by part1, part2 having count(*) > 0 - order by part1 desc limit 1); - """ - - qt_runtime_filter_partition_pruning_two_partition_2 """ - select count(*) from two_partition_tb where part2 = - (select part2 from two_partition_tb - group by part1, part2 having count(*) > 0 - order by part2 desc limit 1); - """ - - qt_runtime_filter_partition_pruning_two_partition_3 """ - select count(*) from two_partition_tb where part1 in - (select part1 from two_partition_tb - group by part1 having count(*) > 0 - order by part1 desc limit 2); - """ - - // Test three partition table (STRING + INT + STRING types) - qt_runtime_filter_partition_pruning_three_partition_1 """ - select count(*) from three_partition_tb where part1 = - (select part1 from three_partition_tb - group by part1, part2, part3 having count(*) > 0 - order by part1 desc limit 1); - """ - - qt_runtime_filter_partition_pruning_three_partition_2 """ - select count(*) from three_partition_tb where part2 = - (select part2 from three_partition_tb - group by part1, part2, part3 having count(*) > 0 - order by part2 desc limit 1); - """ - - qt_runtime_filter_partition_pruning_three_partition_3 """ - select count(*) from three_partition_tb where part3 = - (select part3 from three_partition_tb - group by part1, part2, part3 having count(*) > 0 - order by part3 desc limit 1); - """ - // Test BOOLEAN partition qt_runtime_filter_partition_pruning_boolean_1 """ select count(*) from boolean_partition_tb where part1 = @@ -213,20 +147,20 @@ suite("test_hudi_runtime_filter_partition_pruning", "p2,external,hudi,external_r order by part1 desc limit 2); """ - // Test TIMESTAMP partition - qt_runtime_filter_partition_pruning_timestamp_1 """ - select count(*) from timestamp_partition_tb where part1 = - (select part1 from timestamp_partition_tb - group by part1 having count(*) > 0 - order by part1 desc limit 1); - """ + // // Test TIMESTAMP partition + // qt_runtime_filter_partition_pruning_timestamp_1 """ + // select count(*) from timestamp_partition_tb where part1 = + // (select part1 from timestamp_partition_tb + // group by part1 having count(*) > 0 + // order by part1 desc limit 1); + // """ - qt_runtime_filter_partition_pruning_timestamp_2 """ - select count(*) from timestamp_partition_tb where part1 in - (select part1 from timestamp_partition_tb - group by part1 having count(*) > 0 - order by part1 desc limit 2); - """ + // qt_runtime_filter_partition_pruning_timestamp_2 """ + // select count(*) from timestamp_partition_tb where part1 in + // (select part1 from timestamp_partition_tb + // group by part1 having count(*) > 0 + // order by part1 desc limit 2); + // """ // Additional complex scenarios with multiple filters qt_runtime_filter_partition_pruning_complex_1 """ @@ -246,15 +180,6 @@ suite("test_hudi_runtime_filter_partition_pruning", "p2,external,hudi,external_r group by t2.part2 having count(*) > 1 ); """ - - qt_runtime_filter_partition_pruning_complex_3 """ - select count(*) from three_partition_tb t1 - where (t1.part1, t1.part2) in ( - select t2.part1, t2.part2 from three_partition_tb t2 - where t2.part3 = 'Q1' - group by t2.part1, t2.part2 having count(*) > 0 - ); - """ } try { From 0e09047a1af2e2f6222f2eb23ee8834b812ddd14 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 24 Jul 2025 16:34:45 +0800 Subject: [PATCH 27/43] cache partitionMapInfos --- .../iceberg/source/IcebergScanNode.java | 22 ++++++++------- .../paimon/source/PaimonScanNode.java | 27 +++++++++++-------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 4780d0c8feae4a..d024055b1bbb7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -74,12 +74,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; public class IcebergScanNode extends FileQueryScanNode { @@ -100,7 +100,8 @@ public class IcebergScanNode extends FileQueryScanNode { private long countFromSnapshot; private static final long COUNT_WITH_PARALLEL_SPLITS = 10000; private long targetSplitSize; - private ConcurrentHashMap.KeySetView partitionPathSet; + // This is used to avoid repeatedly calculating partition info map for the same partition data. + private Map> partitionMapInfos; private boolean isPartitionedTable; private int formatVersion; private ExecutionAuthenticator preExecutionAuthenticator; @@ -147,7 +148,7 @@ public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckCol protected void doInitialize() throws UserException { icebergTable = source.getIcebergTable(); targetSplitSize = getRealFileSplitSize(0); - partitionPathSet = ConcurrentHashMap.newKeySet(); + partitionMapInfos = new HashMap<>(); isPartitionedTable = icebergTable.spec().isPartitioned(); formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); preExecutionAuthenticator = source.getCatalog().getExecutionAuthenticator(); @@ -334,11 +335,14 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { if (isPartitionedTable) { PartitionData partitionData = (PartitionData) fileScanTask.file().partition(); if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { - split.setIcebergPartitionValues( - IcebergUtils.getPartitionInfoMap(partitionData, sessionVariable.getTimeZone())); + // If the partition data is not in the map, we need to calculate the partition + Map partitionInfoMap = partitionMapInfos.computeIfAbsent(partitionData, k -> { + return IcebergUtils.getPartitionInfoMap(partitionData, sessionVariable.getTimeZone()); + }); + split.setIcebergPartitionValues(partitionInfoMap); + } else { + partitionMapInfos.put(partitionData, null); } - // Counts the number of partitions read - partitionPathSet.add(partitionData.toString()); } return split; } @@ -368,7 +372,7 @@ private List doGetSplits(int numBackends) throws UserException { throw new UserException(e.getMessage(), e.getCause()); } - selectedPartitionNum = partitionPathSet.size(); + selectedPartitionNum = partitionMapInfos.size(); return splits; } @@ -554,7 +558,7 @@ private void assignCountToSplits(List splits, long totalCount) { @Override public int numApproximateSplits() { - return NUM_SPLITS_PER_PARTITION * partitionPathSet.size() > 0 ? partitionPathSet.size() : 1; + return NUM_SPLITS_PER_PARTITION * partitionMapInfos.size() > 0 ? partitionMapInfos.size() : 1; } private Optional checkNotSupportedException(Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 91b57e1908543d..5d8990214da6b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -44,7 +44,6 @@ import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.paimon.CoreOptions; @@ -64,7 +63,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -266,8 +264,10 @@ public List getSplits(int numBackends) throws UserException { } boolean applyCountPushdown = getPushDownAggNoGroupingOp() == TPushAggOp.COUNT; - // Just for counting the number of selected partitions for this paimon table - Set selectedPartitionValues = Sets.newHashSet(); + // Used to avoid repeatedly calculating partition info map for the same + // partition data. + // And for counting the number of selected partitions for this paimon table. + Map> partitionInfoMaps = new HashMap<>(); // if applyCountPushdown is true, we can't split the DataSplit long realFileSplitSize = getRealFileSplitSize(applyCountPushdown ? Long.MAX_VALUE : 0); for (DataSplit dataSplit : dataSplits) { @@ -275,12 +275,17 @@ public List getSplits(int numBackends) throws UserException { splitStat.setRowCount(dataSplit.rowCount()); BinaryRow partitionValue = dataSplit.partition(); - selectedPartitionValues.add(partitionValue); - // try to get partition info map if EnableRuntimeFilterPartitionPrune is true - Map partitionInfoMap = sessionVariable.isEnableRuntimeFilterPartitionPrune() - ? PaimonUtil.getPartitionInfoMap( - source.getPaimonTable(), partitionValue, sessionVariable.getTimeZone()) - : null; + Map partitionInfoMap = null; + if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { + // If the partition value is not in the map, we need to calculate the partition + // info map and store it in the map. + partitionInfoMap = partitionInfoMaps.computeIfAbsent(partitionValue, k -> { + return PaimonUtil.getPartitionInfoMap( + source.getPaimonTable(), partitionValue, sessionVariable.getTimeZone()); + }); + } else { + partitionInfoMaps.put(partitionValue, null); + } Optional> optRawFiles = dataSplit.convertToRawFiles(); Optional> optDeletionFiles = dataSplit.deletionFiles(); if (applyCountPushdown && dataSplit.mergedRowCountAvailable()) { @@ -355,7 +360,7 @@ public List getSplits(int numBackends) throws UserException { // proportion of each split later. splits.forEach(s -> s.setTargetSplitSize(realFileSplitSize)); - this.selectedPartitionNum = selectedPartitionValues.size(); + this.selectedPartitionNum = partitionInfoMaps.size(); return splits; } From b2fa1965822047a3ffcd4f76d210b7cc79fac281 Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 25 Jul 2025 16:30:57 +0800 Subject: [PATCH 28/43] fix --- be/src/vec/exec/scan/file_scanner.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index c234db96e02962..29f23826667496 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -376,7 +376,8 @@ Status FileScanner::open(RuntimeState* state) { if (_first_scan_range) { RETURN_IF_ERROR(_init_expr_ctxes()); if (_state->query_options().enable_runtime_filter_partition_prune && - !_partition_slot_index_map.empty()) { + (!_partition_slot_index_map.empty() || + _current_range.__isset.data_lake_partition_values)) { _init_runtime_filter_partition_prune_ctxs(); _init_runtime_filter_partition_prune_block(); } @@ -917,6 +918,7 @@ Status FileScanner::_get_next_reader() { // try to get the partition columns from the range RETURN_IF_ERROR(_generate_partition_columns()); + // try to get the data lake partition columns from the range RETURN_IF_ERROR(_generate_data_lake_partition_columns()); const auto& partition_col_descs = !_partition_col_descs.empty() From 204f7999f1794dae7f9df41568d4c061d2d250db Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 29 Jul 2025 11:15:45 +0800 Subject: [PATCH 29/43] fix hudi case --- ..._hudi_runtime_filter_partition_pruning.out | 12 +++++++++ ...di_runtime_filter_partition_pruning.groovy | 26 +++++++++---------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out index 0d6c5cd9e064da..abf14662aba9ad 100644 --- a/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.out @@ -44,6 +44,12 @@ -- !runtime_filter_partition_pruning_date_2 -- 4 +-- !runtime_filter_partition_pruning_timestamp_1 -- +2 + +-- !runtime_filter_partition_pruning_timestamp_2 -- +4 + -- !runtime_filter_partition_pruning_complex_1 -- 7 @@ -95,6 +101,12 @@ -- !runtime_filter_partition_pruning_date_2 -- 4 +-- !runtime_filter_partition_pruning_timestamp_1 -- +2 + +-- !runtime_filter_partition_pruning_timestamp_2 -- +4 + -- !runtime_filter_partition_pruning_complex_1 -- 7 diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy index fb6e3e26bf0647..de4757f7c6f062 100644 --- a/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_runtime_filter_partition_pruning.groovy @@ -147,20 +147,20 @@ suite("test_hudi_runtime_filter_partition_pruning", "p2,external,hudi,external_r order by part1 desc limit 2); """ - // // Test TIMESTAMP partition - // qt_runtime_filter_partition_pruning_timestamp_1 """ - // select count(*) from timestamp_partition_tb where part1 = - // (select part1 from timestamp_partition_tb - // group by part1 having count(*) > 0 - // order by part1 desc limit 1); - // """ + // Test TIMESTAMP partition + qt_runtime_filter_partition_pruning_timestamp_1 """ + select count(*) from timestamp_partition_tb where part1 = + (select part1 from timestamp_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 1); + """ - // qt_runtime_filter_partition_pruning_timestamp_2 """ - // select count(*) from timestamp_partition_tb where part1 in - // (select part1 from timestamp_partition_tb - // group by part1 having count(*) > 0 - // order by part1 desc limit 2); - // """ + qt_runtime_filter_partition_pruning_timestamp_2 """ + select count(*) from timestamp_partition_tb where part1 in + (select part1 from timestamp_partition_tb + group by part1 having count(*) > 0 + order by part1 desc limit 2); + """ // Additional complex scenarios with multiple filters qt_runtime_filter_partition_pruning_complex_1 """ From 6f2ad12c809921af04bbc19a72e9bc5e344fc865 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 31 Jul 2025 22:20:26 +0800 Subject: [PATCH 30/43] fix be --- be/src/vec/exec/scan/file_scanner.cpp | 92 +++++++++++---------------- be/src/vec/exec/scan/file_scanner.h | 17 +---- gensrc/thrift/PlanNodes.thrift | 2 - 3 files changed, 39 insertions(+), 72 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 29f23826667496..46e69d06cfd425 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -206,7 +206,8 @@ Status FileScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conjun bool FileScanner::_check_partition_prune_expr(const VExprSPtr& expr) { if (expr->is_slot_ref()) { auto* slot_ref = static_cast(expr.get()); - return _partition_col_names.find(slot_ref->expr_name()) != _partition_col_names.end(); + return _partition_slot_index_map.find(slot_ref->slot_id()) != + _partition_slot_index_map.end(); } if (expr->is_literal()) { return true; @@ -241,19 +242,16 @@ void FileScanner::_init_runtime_filter_partition_prune_block() { } } -Status FileScanner::_process_runtime_filters_partition_prune( - const std::unordered_map>& - partition_col_descs, - bool& can_filter_all) { +Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_all) { SCOPED_TIMER(_runtime_filter_partition_prune_timer); - if (_runtime_filter_partition_prune_ctxs.empty() || partition_col_descs.empty()) { + if (_runtime_filter_partition_prune_ctxs.empty() || _partition_col_descs.empty()) { return Status::OK(); } size_t partition_value_column_size = 1; // 1. Get partition key values to string columns. std::unordered_map partition_slot_id_to_column; - for (auto const& partition_col_desc : partition_col_descs) { + for (auto const& partition_col_desc : _partition_col_descs) { const auto& [partition_value, partition_slot_desc] = partition_col_desc.second; auto test_serde = partition_slot_desc->get_data_type_ptr()->get_serde(); auto partition_value_column = partition_slot_desc->get_data_type_ptr()->create_column(); @@ -376,8 +374,7 @@ Status FileScanner::open(RuntimeState* state) { if (_first_scan_range) { RETURN_IF_ERROR(_init_expr_ctxes()); if (_state->query_options().enable_runtime_filter_partition_prune && - (!_partition_slot_index_map.empty() || - _current_range.__isset.data_lake_partition_values)) { + !_partition_slot_index_map.empty()) { _init_runtime_filter_partition_prune_ctxs(); _init_runtime_filter_partition_prune_block(); } @@ -916,32 +913,26 @@ Status FileScanner::_get_next_reader() { const TFileRangeDesc& range = _current_range; _current_range_path = range.path; - // try to get the partition columns from the range - RETURN_IF_ERROR(_generate_partition_columns()); - // try to get the data lake partition columns from the range - RETURN_IF_ERROR(_generate_data_lake_partition_columns()); + if (!_partition_slot_descs.empty()) { + // we need get partition columns first for runtime filter partition pruning + RETURN_IF_ERROR(_generate_partition_columns()); - const auto& partition_col_descs = !_partition_col_descs.empty() - ? _partition_col_descs - : _data_lake_partition_col_descs; - - if (_state->query_options().enable_runtime_filter_partition_prune && - !partition_col_descs.empty()) { - // if enable_runtime_filter_partition_prune is true, we need to check whether this range can be filtered out - // by runtime filter partition prune - if (_push_down_conjuncts.size() < _conjuncts.size()) { - // there are new runtime filters, need to re-init runtime filter partition pruning ctxs - _init_runtime_filter_partition_prune_ctxs(); - } + if (_state->query_options().enable_runtime_filter_partition_prune) { + // if enable_runtime_filter_partition_prune is true, we need to check whether this range can be filtered out + // by runtime filter partition prune + if (_push_down_conjuncts.size() < _conjuncts.size()) { + // there are new runtime filters, need to re-init runtime filter partition pruning ctxs + _init_runtime_filter_partition_prune_ctxs(); + } - bool can_filter_all = false; - RETURN_IF_ERROR( - _process_runtime_filters_partition_prune(partition_col_descs, can_filter_all)); - if (can_filter_all) { - // this range can be filtered out by runtime filter partition pruning - // so we need to skip this range - COUNTER_UPDATE(_runtime_filter_partition_pruned_range_counter, 1); - continue; + bool can_filter_all = false; + RETURN_IF_ERROR(_process_runtime_filters_partition_prune(can_filter_all)); + if (can_filter_all) { + // this range can be filtered out by runtime filter partition pruning + // so we need to skip this range + COUNTER_UPDATE(_runtime_filter_partition_pruned_range_counter, 1); + continue; + } } } @@ -1393,6 +1384,7 @@ Status FileScanner::prepare_for_read_lines(const TFileRangeDesc& range) { _push_down_conjuncts.clear(); _not_single_slot_filter_conjuncts.clear(); _slot_id_to_filter_conjuncts.clear(); + _partition_slots_need_fill_from_path.clear(); _kv_cache = nullptr; return Status::OK(); } @@ -1470,6 +1462,13 @@ Status FileScanner::_generate_partition_columns() { if (range.__isset.columns_from_path && !_partition_slot_descs.empty()) { for (const auto& slot_desc : _partition_slot_descs) { if (slot_desc) { + // Only generate partition column descriptions for slots that need to be filled from path + // For table formats like Iceberg/Paimon, partition values are already in the file + if (_partition_slots_need_fill_from_path.find(slot_desc->id()) == + _partition_slots_need_fill_from_path.end()) { + continue; + } + auto it = _partition_slot_index_map.find(slot_desc->id()); if (it == std::end(_partition_slot_index_map)) { return Status::InternalError("Unknown source slot descriptor, slot_id={}", @@ -1489,21 +1488,6 @@ Status FileScanner::_generate_partition_columns() { return Status::OK(); } -Status FileScanner::_generate_data_lake_partition_columns() { - _data_lake_partition_col_descs.clear(); - if (_current_range.__isset.data_lake_partition_values) { - const auto& partition_values = _current_range.data_lake_partition_values; - for (const auto& [key, value] : partition_values) { - if (_all_col_name_to_slot_desc.find(key) == _all_col_name_to_slot_desc.end()) { - continue; // skip if the key is not in the slot desc map - } - const auto* slot_desc = _all_col_name_to_slot_desc[key]; - _data_lake_partition_col_descs.emplace(key, std::make_tuple(value, slot_desc)); - } - } - return Status::OK(); -} - Status FileScanner::_generate_missing_columns() { _missing_col_descs.clear(); if (!_missing_cols.empty()) { @@ -1547,17 +1531,10 @@ Status FileScanner::_init_expr_ctxes() { if (!key_map.empty()) { for (size_t i = 0; i < key_map.size(); i++) { partition_name_to_key_index_map.emplace(key_map[i], i); - _partition_col_names.insert(key_map[i]); } } } - if (_current_range.__isset.data_lake_partition_values) { - for (const auto& partition_value : _current_range.data_lake_partition_values) { - _partition_col_names.insert(partition_value.first); - } - } - _num_of_columns_from_file = _params->num_of_columns_from_file; for (const auto& slot_info : _params->required_slots) { @@ -1576,6 +1553,10 @@ Status FileScanner::_init_expr_ctxes() { _file_col_names.push_back(it->second->col_name()); } else { _partition_slot_descs.emplace_back(it->second); + // Only traditional partition columns (is_file_slot=false) need to be filled from path + // For table formats like Iceberg/Paimon, partition columns have is_file_slot=true + // because their values are stored in files, not derived from path + _partition_slots_need_fill_from_path.insert(slot_id); if (_is_load) { auto iti = full_src_index_map.find(slot_id); _partition_slot_index_map.emplace(slot_id, iti->second - _num_of_columns_from_file); @@ -1584,7 +1565,6 @@ Status FileScanner::_init_expr_ctxes() { _partition_slot_index_map.emplace(slot_id, kit->second); } } - _all_col_name_to_slot_desc.emplace(it->second->col_name(), it->second); } // set column name to default value expr map diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h index ddae84d97e89df..4dca91992ade42 100644 --- a/be/src/vec/exec/scan/file_scanner.h +++ b/be/src/vec/exec/scan/file_scanner.h @@ -135,6 +135,8 @@ class FileScanner : public Scanner { std::vector _partition_slot_descs; // Partition slot id to index in _partition_slot_descs std::unordered_map _partition_slot_index_map; + // Partition slots that need to be filled from path + std::unordered_set _partition_slots_need_fill_from_path; // created from param.expr_of_dest_slot // For query, it saves default value expr of all dest columns, or nullptr for NULL. // For load, it saves conversion expr/default value of all dest columns. @@ -192,15 +194,6 @@ class FileScanner : public Scanner { _partition_col_descs; std::unordered_map _missing_col_descs; - // store all slot descriptors, used for initializing runtime filter partition prune context - std::unordered_map _all_col_name_to_slot_desc; - // store data lake partition column descriptors - std::unordered_map> - _data_lake_partition_col_descs; - - // partition column names, used for initializing runtime filter partition prune context - std::set _partition_col_names; - // idx of skip_bitmap_col in _input_tuple_desc int32_t _skip_bitmap_col_idx {-1}; int32_t _sequence_map_col_uid {-1}; @@ -251,15 +244,11 @@ class FileScanner : public Scanner { Status _truncate_char_or_varchar_columns(Block* block); void _truncate_char_or_varchar_column(Block* block, int idx, int len); Status _generate_partition_columns(); - Status _generate_data_lake_partition_columns(); Status _generate_missing_columns(); bool _check_partition_prune_expr(const VExprSPtr& expr); void _init_runtime_filter_partition_prune_ctxs(); void _init_runtime_filter_partition_prune_block(); - Status _process_runtime_filters_partition_prune( - const std::unordered_map>& - partition_col_descs, - bool& is_partition_pruned); + Status _process_runtime_filters_partition_prune(bool& is_partition_pruned); Status _process_conjuncts_for_dict_filter(); Status _process_late_arrival_conjuncts(); void _get_slot_ids(VExpr* expr, std::vector* slot_ids); diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 2010d2254455b7..bbdc414cbd71ed 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -493,8 +493,6 @@ struct TFileRangeDesc { 12: optional string fs_name 13: optional TFileFormatType format_type; 14: optional i64 self_split_weight - // partition values for data lake table format like iceberg/hudi/paimon/lakesoul - 15: optional map data_lake_partition_values; } struct TSplitSource { From 4c5199c2ac32d83f647339a0259880c44027001d Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 31 Jul 2025 22:24:44 +0800 Subject: [PATCH 31/43] fix be --- be/src/vec/exec/scan/file_scanner.cpp | 12 ------------ be/src/vec/exec/scan/file_scanner.h | 2 -- 2 files changed, 14 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 46e69d06cfd425..a650836573ae2e 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -1384,7 +1384,6 @@ Status FileScanner::prepare_for_read_lines(const TFileRangeDesc& range) { _push_down_conjuncts.clear(); _not_single_slot_filter_conjuncts.clear(); _slot_id_to_filter_conjuncts.clear(); - _partition_slots_need_fill_from_path.clear(); _kv_cache = nullptr; return Status::OK(); } @@ -1462,13 +1461,6 @@ Status FileScanner::_generate_partition_columns() { if (range.__isset.columns_from_path && !_partition_slot_descs.empty()) { for (const auto& slot_desc : _partition_slot_descs) { if (slot_desc) { - // Only generate partition column descriptions for slots that need to be filled from path - // For table formats like Iceberg/Paimon, partition values are already in the file - if (_partition_slots_need_fill_from_path.find(slot_desc->id()) == - _partition_slots_need_fill_from_path.end()) { - continue; - } - auto it = _partition_slot_index_map.find(slot_desc->id()); if (it == std::end(_partition_slot_index_map)) { return Status::InternalError("Unknown source slot descriptor, slot_id={}", @@ -1553,10 +1545,6 @@ Status FileScanner::_init_expr_ctxes() { _file_col_names.push_back(it->second->col_name()); } else { _partition_slot_descs.emplace_back(it->second); - // Only traditional partition columns (is_file_slot=false) need to be filled from path - // For table formats like Iceberg/Paimon, partition columns have is_file_slot=true - // because their values are stored in files, not derived from path - _partition_slots_need_fill_from_path.insert(slot_id); if (_is_load) { auto iti = full_src_index_map.find(slot_id); _partition_slot_index_map.emplace(slot_id, iti->second - _num_of_columns_from_file); diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h index 4dca91992ade42..07c0d7e41dc3e0 100644 --- a/be/src/vec/exec/scan/file_scanner.h +++ b/be/src/vec/exec/scan/file_scanner.h @@ -135,8 +135,6 @@ class FileScanner : public Scanner { std::vector _partition_slot_descs; // Partition slot id to index in _partition_slot_descs std::unordered_map _partition_slot_index_map; - // Partition slots that need to be filled from path - std::unordered_set _partition_slots_need_fill_from_path; // created from param.expr_of_dest_slot // For query, it saves default value expr of all dest columns, or nullptr for NULL. // For load, it saves conversion expr/default value of all dest columns. From f81629a96207b447c5756a5cea3e103eba6817c8 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 31 Jul 2025 23:11:46 +0800 Subject: [PATCH 32/43] fix fe --- .../doris/datasource/iceberg/IcebergUtils.java | 8 ++++---- .../iceberg/source/IcebergScanNode.java | 16 ++++++++++++++-- .../doris/datasource/paimon/PaimonUtil.java | 8 ++++---- .../datasource/paimon/source/PaimonScanNode.java | 16 ++++++++++++++-- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index a9e317c31633df..48371e7fa7673f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -49,6 +49,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CacheException; @@ -619,7 +620,8 @@ public static Map getPartitionInfoMap(PartitionData partitionDat String partitionString = serializePartitionValue(field.type(), value, timeZone); partitionInfoMap.put(field.name(), partitionString); } catch (UnsupportedOperationException e) { - LOG.warn("Failed to serialize partition value for field {}: {}", field.name(), e.getMessage()); + LOG.warn("Failed to serialize Iceberg table partition value for field {}: {}", field.name(), + e.getMessage()); return null; } } @@ -628,14 +630,12 @@ public static Map getPartitionInfoMap(PartitionData partitionDat private static String serializePartitionValue(org.apache.iceberg.types.Type type, Object value, String timeZone) { if (value == null) { - return "\\N"; + return FeConstants.null_string; } switch (type.typeId()) { case BOOLEAN: case INTEGER: case LONG: - case FLOAT: - case DOUBLE: case STRING: case UUID: case DECIMAL: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d024055b1bbb7a..1d7ce7fcde4be5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -202,7 +202,17 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli } } tableFormatFileDesc.setIcebergParams(fileDesc); - rangeDesc.setDataLakePartitionValues(icebergSplit.getIcebergPartitionValues()); + Map partitionValues = icebergSplit.getIcebergPartitionValues(); + if (partitionValues != null) { + List formPathKeys = new ArrayList<>(); + List formPathValues = new ArrayList<>(); + for (Map.Entry entry : partitionValues.entrySet()) { + formPathKeys.add(entry.getKey()); + formPathValues.add(entry.getValue()); + } + rangeDesc.setColumnsFromPathKeys(formPathKeys); + rangeDesc.setColumnsFromPath(formPathValues); + } rangeDesc.setTableFormatParams(tableFormatFileDesc); } @@ -339,7 +349,9 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { Map partitionInfoMap = partitionMapInfos.computeIfAbsent(partitionData, k -> { return IcebergUtils.getPartitionInfoMap(partitionData, sessionVariable.getTimeZone()); }); - split.setIcebergPartitionValues(partitionInfoMap); + if (partitionInfoMap != null) { + split.setIcebergPartitionValues(partitionInfoMap); + } } else { partitionMapInfos.put(partitionData, null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 76ec6215bfe1ec..1809a9a7083a10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.hive.HiveUtil; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; @@ -401,7 +402,8 @@ public static Map getPartitionInfoMap(Table table, BinaryRow par partitionValuesArray[i], timeZone); partitionInfoMap.put(partitionKeys.get(i), partitionValue); } catch (UnsupportedOperationException e) { - LOG.warn("Failed to serialize partition value for key {}: {}", partitionKeys.get(i), e.getMessage()); + LOG.warn("Failed to serialize table {} partition value for key {}: {}", table.name(), + partitionKeys.get(i), e.getMessage()); return null; } } @@ -411,14 +413,12 @@ public static Map getPartitionInfoMap(Table table, BinaryRow par private static String serializePartitionValue(org.apache.paimon.types.DataType type, Object value, String timeZone) { if (value == null) { - return "\\N"; + return FeConstants.null_string; } switch (type.getTypeRoot()) { case BOOLEAN: case INTEGER: case BIGINT: - case FLOAT: - case DOUBLE: case SMALLINT: case TINYINT: case DECIMAL: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 5d8990214da6b1..84d98e48c9a122 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -241,7 +241,17 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) tableFormatFileDesc.setTableLevelRowCount(paimonSplit.getRowCount().get()); } tableFormatFileDesc.setPaimonParams(fileDesc); - rangeDesc.setDataLakePartitionValues(paimonSplit.getPaimonPartitionValues()); + Map partitionValues = paimonSplit.getPaimonPartitionValues(); + if (partitionValues != null) { + List formPathKeys = new ArrayList<>(); + List formPathValues = new ArrayList<>(); + for (Map.Entry entry : partitionValues.entrySet()) { + formPathKeys.add(entry.getKey()); + formPathValues.add(entry.getValue()); + } + rangeDesc.setColumnsFromPathKeys(formPathKeys); + rangeDesc.setColumnsFromPath(formPathValues); + } rangeDesc.setTableFormatParams(tableFormatFileDesc); } @@ -292,7 +302,9 @@ public List getSplits(int numBackends) throws UserException { splitStat.setMergedRowCount(dataSplit.mergedRowCount()); PaimonSplit split = new PaimonSplit(dataSplit); split.setRowCount(dataSplit.mergedRowCount()); - split.setPaimonPartitionValues(partitionInfoMap); + if (partitionInfoMap != null) { + split.setPaimonPartitionValues(partitionInfoMap); + } pushDownCountSplits.add(split); pushDownCountSum += dataSplit.mergedRowCount(); } else if (!forceJniScanner && supportNativeReader(optRawFiles)) { From 917b8b95d57fc1a5f249a45ec648f062f0662098 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 31 Jul 2025 23:24:09 +0800 Subject: [PATCH 33/43] fix hudi --- .../datasource/hudi/source/HudiScanNode.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index d79255bf421d0e..69af4ebfab3f98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -307,7 +307,17 @@ private void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { } } tableFormatFileDesc.setHudiParams(fileDesc); - rangeDesc.setDataLakePartitionValues(hudiSplit.getHudiPartitionValues()); + Map partitionValues = hudiSplit.getHudiPartitionValues(); + if (partitionValues != null) { + List formPathKeys = new ArrayList<>(); + List formPathValues = new ArrayList<>(); + for (Map.Entry entry : partitionValues.entrySet()) { + formPathKeys.add(entry.getKey()); + formPathValues.add(entry.getValue()); + } + rangeDesc.setColumnsFromPathKeys(formPathKeys); + rangeDesc.setColumnsFromPath(formPathValues); + } rangeDesc.setTableFormatParams(tableFormatFileDesc); } @@ -373,6 +383,11 @@ private void getPartitionSplits(HivePartition partition, List splits) thr new StoragePath(partition.getPath())); } + Map partitionValues = null; + if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { + partitionValues = HudiUtils.getPartitionInfoMap(hmsTable, partition); + } + if (canUseNativeReader()) { fsView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { noLogsSplitNum.incrementAndGet(); @@ -384,8 +399,8 @@ private void getPartitionSplits(HivePartition partition, List splits) thr HudiSplit hudiSplit = new HudiSplit(locationPath, 0, fileSize, fileSize, new String[0], partition.getPartitionValues()); hudiSplit.setTableFormatType(TableFormatType.HUDI); - if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { - hudiSplit.setHudiPartitionValues(HudiUtils.getPartitionInfoMap(hmsTable, partition)); + if (partitionValues != null) { + hudiSplit.setHudiPartitionValues(partitionValues); } splits.add(hudiSplit); }); From a80d81d25e4035e0430324ca0cd447132817ab1d Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 31 Jul 2025 22:52:44 +0800 Subject: [PATCH 34/43] intro _fill_partition_from_path --- be/src/vec/exec/scan/file_scanner.cpp | 18 ++++++++++++++++-- be/src/vec/exec/scan/file_scanner.h | 2 ++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index a650836573ae2e..73bd990864954d 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -602,6 +602,9 @@ Status FileScanner::_cast_to_input_block(Block* block) { } Status FileScanner::_fill_columns_from_path(size_t rows) { + if (!_fill_partition_from_path) { + return Status::OK(); + } DataTypeSerDe::FormatOptions _text_formatOptions; for (auto& kv : _partition_col_descs) { auto doris_column = _src_block_ptr->get_by_name(kv.first).column; @@ -1332,7 +1335,12 @@ Status FileScanner::_set_fill_or_truncate_columns(bool need_to_get_parsed_schema } RETURN_IF_ERROR(_generate_missing_columns()); - RETURN_IF_ERROR(_cur_reader->set_fill_columns(_partition_col_descs, _missing_col_descs)); + if (_fill_partition_from_path) { + RETURN_IF_ERROR(_cur_reader->set_fill_columns(_partition_col_descs, _missing_col_descs)); + } else { + // If the partition columns are not from path, we only fill the missing columns. + RETURN_IF_ERROR(_cur_reader->set_fill_columns({}, _missing_col_descs)); + } if (VLOG_NOTICE_IS_ON && !_missing_cols.empty() && _is_load) { fmt::memory_buffer col_buf; for (auto& col : _missing_cols) { @@ -1540,10 +1548,16 @@ Status FileScanner::_init_expr_ctxes() { _row_id_column_iterator_pair.second = _default_val_row_desc->get_column_id(slot_id); continue; } - if (slot_info.is_file_slot) { + if (partition_name_to_key_index_map.find(it->second->col_name()) == + partition_name_to_key_index_map.end()) { _file_slot_descs.emplace_back(it->second); _file_col_names.push_back(it->second->col_name()); } else { + if (slot_info.is_file_slot) { + // If there is slot which is both a partition column and a file column, + // we should not fill the partition column from path. + _fill_partition_from_path = false; + } _partition_slot_descs.emplace_back(it->second); if (_is_load) { auto iti = full_src_index_map.find(slot_id); diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h index 07c0d7e41dc3e0..3556897b6274a3 100644 --- a/be/src/vec/exec/scan/file_scanner.h +++ b/be/src/vec/exec/scan/file_scanner.h @@ -188,6 +188,8 @@ class FileScanner : public Scanner { std::unique_ptr _file_reader_stats; std::unique_ptr _io_ctx; + // Whether to fill partition columns from path, default is true. + bool _fill_partition_from_path = true; std::unordered_map> _partition_col_descs; std::unordered_map _missing_col_descs; From f2621685eb94eea28ea2ccd321bf72706b21ae92 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 31 Jul 2025 23:47:48 +0800 Subject: [PATCH 35/43] fix build --- .../apache/doris/datasource/hudi/source/HudiScanNode.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 69af4ebfab3f98..c30cffad98244d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -383,10 +383,9 @@ private void getPartitionSplits(HivePartition partition, List splits) thr new StoragePath(partition.getPath())); } - Map partitionValues = null; - if (sessionVariable.isEnableRuntimeFilterPartitionPrune()) { - partitionValues = HudiUtils.getPartitionInfoMap(hmsTable, partition); - } + final Map partitionValues = sessionVariable.isEnableRuntimeFilterPartitionPrune() + ? HudiUtils.getPartitionInfoMap(hmsTable, partition) + : null; if (canUseNativeReader()) { fsView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { From d2439b5fa090ee45f1c6f9d8cda06efa063cb101 Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 1 Aug 2025 00:33:46 +0800 Subject: [PATCH 36/43] fix cases --- be/src/vec/exec/scan/file_scanner.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 73bd990864954d..3997dd7b5ae485 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -1548,15 +1548,24 @@ Status FileScanner::_init_expr_ctxes() { _row_id_column_iterator_pair.second = _default_val_row_desc->get_column_id(slot_id); continue; } - if (partition_name_to_key_index_map.find(it->second->col_name()) == - partition_name_to_key_index_map.end()) { + + if (slot_info.is_file_slot) { _file_slot_descs.emplace_back(it->second); _file_col_names.push_back(it->second->col_name()); - } else { + } + + if (partition_name_to_key_index_map.find(it->second->col_name()) != + partition_name_to_key_index_map.end()) { if (slot_info.is_file_slot) { // If there is slot which is both a partition column and a file column, // we should not fill the partition column from path. _fill_partition_from_path = false; + } else if (!_fill_partition_from_path) { + // This should not happen + return Status::InternalError( + "Partition column {} is not a file column, but there is already a column " + "which is both a partition column and a file column.", + it->second->col_name()); } _partition_slot_descs.emplace_back(it->second); if (_is_load) { From bacf77835333138c76ef544bbf2967dc046e809d Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 1 Aug 2025 16:17:06 +0800 Subject: [PATCH 37/43] fix bug about null partition --- .../iceberg/run18.sql | 33 +++++++++--- .../paimon/run06.sql | 31 ++++++++--- .../datasource/iceberg/IcebergUtils.java | 18 +++++-- .../doris/datasource/paimon/PaimonUtil.java | 21 ++++++-- ...eberg_runtime_filter_partition_pruning.out | 54 +++++++++++++++++++ ...aimon_runtime_filter_partition_pruning.out | 48 +++++++++++++++++ ...rg_runtime_filter_partition_pruning.groovy | 45 ++++++++++++++++ ...on_runtime_filter_partition_pruning.groovy | 40 ++++++++++++++ 8 files changed, 271 insertions(+), 19 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql index 23adcc82aa0d78..77bfad8fdb0587 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql @@ -21,7 +21,8 @@ VALUES (1, 'Alice', DATE '2024-01-01'), DATE '2024-02-01' ), (4, 'David', DATE '2024-02-01'), - (5, 'Eve', DATE '2024-03-01'); + (5, 'Eve', DATE '2024-03-01'), + (6, 'Null Date', NULL); -- Partition by integer type CREATE TABLE int_partitioned ( @@ -37,7 +38,8 @@ VALUES (1, 'Product A', 1), (2, 'Product B', 1), (3, 'Product C', 2), (4, 'Product D', 2), - (5, 'Product E', 3); + (5, 'Product E', 3), + (6, 'Null Int', NULL); -- Partition by float type CREATE TABLE float_partitioned ( @@ -53,7 +55,8 @@ VALUES (1, 'Item 1', 10.5), (2, 'Item 2', 20.75), (3, 'Item 3', 30.0), (4, 'Item 4', 40.25), - (5, 'Item 5', 50.5); + (5, 'Item 5', 50.5), + (6, 'Null Float', NULL); -- Partition by string type CREATE TABLE string_partitioned ( @@ -70,7 +73,8 @@ VALUES (1, 'User1', 'North America'), (3, 'User3', 'Europe'), (4, 'User4', 'Europe'), (5, 'User5', 'Asia'), - (6, 'User6', 'Asia'); + (6, 'User6', 'Asia'), + (7, 'Null String', NULL); -- Partition by timestamp type CREATE TABLE timestamp_partitioned ( @@ -106,6 +110,11 @@ VALUES ( 5, 'Event5', TIMESTAMP '2024-01-16 16:00:00' + ), + ( + 6, + 'Null Timestamp', + NULL ); -- Partition by timestamp_ntz type @@ -142,6 +151,11 @@ VALUES ( 5, 'Event5', TIMESTAMP_NTZ '2024-01-16 16:00:00' + ), + ( + 6, + 'Null Timestamp NTZ', + NULL ); -- Partition by boolean type @@ -158,7 +172,8 @@ VALUES (1, 'Active User', true), (2, 'Active Admin', true), (3, 'Inactive User', false), (4, 'Inactive Guest', false), - (5, 'Active Manager', true); + (5, 'Active Manager', true), + (6, 'Null Boolean', NULL); -- Partition by decimal type CREATE TABLE decimal_partitioned ( @@ -176,7 +191,8 @@ VALUES (1, 'Item A', 125.50, 10.50), (3, 'Item C', 89.99, 25.25), (4, 'Item D', 156.80, 25.25), (5, 'Item E', 299.95, 50.00), - (6, 'Item F', 399.99, 50.00); + (6, 'Item F', 399.99, 50.00), + (7, 'Null Decimal', 0.0, NULL); -- Partition by binary type CREATE TABLE binary_partitioned ( id BIGINT, @@ -211,4 +227,9 @@ VALUES ( 5, 'Binary Data 5', CAST('11110000' AS BINARY) + ), + ( + 6, + 'Null Binary', + NULL ); \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql index bcadfd7be3564b..bf5e436e9d8ca9 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -25,7 +25,8 @@ VALUES (1, 'Alice', DATE '2024-01-01'), DATE '2024-02-01' ), (4, 'David', DATE '2024-02-01'), - (5, 'Eve', DATE '2024-03-01'); + (5, 'Eve', DATE '2024-03-01'), + (6, 'Null Date', NULL); -- Partition by timestamp type CREATE TABLE timestamp_partitioned ( @@ -61,6 +62,11 @@ VALUES ( 5, 'Event5', TIMESTAMP '2024-01-16 16:00:00' + ), + ( + 6, + 'Null Timestamp', + NULL ); -- Partition by integer type @@ -77,7 +83,8 @@ VALUES (1, 'Product A', 1), (2, 'Product B', 1), (3, 'Product C', 2), (4, 'Product D', 2), - (5, 'Product E', 3); + (5, 'Product E', 3), + (6, 'Null Int', NULL); -- Partition by bigint type CREATE TABLE bigint_partitioned ( @@ -93,7 +100,8 @@ VALUES (1, 'Item 1', 100), (2, 'Item 2', 100), (3, 'Item 3', 200), (4, 'Item 4', 200), - (5, 'Item 5', 300); + (5, 'Item 5', 300), + (6, 'Null Bigint', NULL); -- Partition by string type CREATE TABLE string_partitioned ( @@ -110,7 +118,8 @@ VALUES (1, 'User1', 'North America'), (3, 'User3', 'Europe'), (4, 'User4', 'Europe'), (5, 'User5', 'Asia'), - (6, 'User6', 'Asia'); + (6, 'User6', 'Asia'), + (7, 'Null String', NULL); -- Partition by boolean type CREATE TABLE boolean_partitioned ( @@ -126,7 +135,8 @@ VALUES (1, 'Active User', true), (2, 'Active Admin', true), (3, 'Inactive User', false), (4, 'Inactive Guest', false), - (5, 'Active Manager', true); + (5, 'Active Manager', true), + (6, 'Null Boolean', NULL); -- Partition by decimal type CREATE TABLE decimal_partitioned ( @@ -144,7 +154,8 @@ VALUES (1, 'Item A', 125.50, 10.50), (3, 'Item C', 89.99, 25.25), (4, 'Item D', 156.80, 25.25), (5, 'Item E', 299.95, 50.00), - (6, 'Item F', 399.99, 50.00); + (6, 'Item F', 399.99, 50.00), + (7, 'Null Decimal', 0.0, NULL); -- Partition by binary type CREATE TABLE binary_partitioned ( @@ -180,6 +191,11 @@ VALUES ( 5, 'Binary Data 5', CAST('binary3' AS BINARY) + ), + ( + 6, + 'Null Binary', + NULL ); -- Partition by float type @@ -195,4 +211,5 @@ VALUES (1, 'Float Data 1', 1.5), (2, 'Float Data 2', 1.5), (3, 'Float Data 3', 2.5), (4, 'Float Data 4', 2.5), - (5, 'Float Data 5', 3.5); \ No newline at end of file + (5, 'Float Data 5', 3.5), + (6, 'Null Float', NULL); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 48371e7fa7673f..aba7d0da928c1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -629,9 +629,6 @@ public static Map getPartitionInfoMap(PartitionData partitionDat } private static String serializePartitionValue(org.apache.iceberg.types.Type type, Object value, String timeZone) { - if (value == null) { - return FeConstants.null_string; - } switch (type.typeId()) { case BOOLEAN: case INTEGER: @@ -639,24 +636,39 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type case STRING: case UUID: case DECIMAL: + if (value == null) { + return FeConstants.null_string; + } return value.toString(); case FIXED: case BINARY: + if (value == null) { + return FeConstants.null_string; + } // Fixed and binary types are stored as ByteBuffer ByteBuffer buffer = (ByteBuffer) value; byte[] res = new byte[buffer.limit()]; buffer.get(res); return new String(res, StandardCharsets.UTF_8); case DATE: + if (value == null) { + return FeConstants.null_string; + } // Iceberg date is stored as days since epoch (1970-01-01) LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME: + if (value == null) { + return FeConstants.null_string; + } // Iceberg time is stored as microseconds since midnight long micros = (Long) value; LocalTime time = LocalTime.ofNanoOfDay(micros * 1000); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP: + if (value == null) { + return FeConstants.null_string; + } // Iceberg timestamp is stored as microseconds since epoch // (1970-01-01T00:00:00) long timestampMicros = (Long) value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 1809a9a7083a10..319f71fb775711 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -412,9 +412,6 @@ public static Map getPartitionInfoMap(Table table, BinaryRow par private static String serializePartitionValue(org.apache.paimon.types.DataType type, Object value, String timeZone) { - if (value == null) { - return FeConstants.null_string; - } switch (type.getTypeRoot()) { case BOOLEAN: case INTEGER: @@ -424,23 +421,41 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t case DECIMAL: case VARCHAR: case CHAR: + if (value == null) { + return FeConstants.null_string; + } return value.toString(); case BINARY: case VARBINARY: + if (value == null) { + return FeConstants.null_string; + } return new String((byte[]) value, StandardCharsets.UTF_8); case DATE: + if (value == null) { + return FeConstants.null_string; + } // Paimon date is stored as days since epoch LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME_WITHOUT_TIME_ZONE: + if (value == null) { + return FeConstants.null_string; + } // Paimon time is stored as microseconds since midnight in utc long micros = (Long) value; LocalTime time = LocalTime.ofNanoOfDay(micros * 1000); return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP_WITHOUT_TIME_ZONE: + if (value == null) { + return FeConstants.null_string; + } // Paimon timestamp is stored as Timestamp type in utc return ((Timestamp) value).toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + if (value == null) { + return FeConstants.null_string; + } // Paimon timestamp with local time zone is stored as Timestamp type in utc Timestamp timestamp = (Timestamp) value; return timestamp.toLocalDateTime() diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index d1e5dce9e57826..67d91cb79ee066 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -8,6 +8,9 @@ -- !runtime_filter_partition_pruning_decimal3 -- 2 +-- !runtime_filter_partition_pruning_decimal_in_null -- +2 + -- !runtime_filter_partition_pruning_int1 -- 1 @@ -17,30 +20,45 @@ -- !runtime_filter_partition_pruning_int3 -- 1 +-- !runtime_filter_partition_pruning_int_in_null -- +1 + -- !runtime_filter_partition_pruning_string1 -- 2 -- !runtime_filter_partition_pruning_string2 -- 4 +-- !runtime_filter_partition_pruning_string_in_null -- +2 + -- !runtime_filter_partition_pruning_date1 -- 1 -- !runtime_filter_partition_pruning_date2 -- 3 +-- !runtime_filter_partition_pruning_date_in_null -- +1 + -- !runtime_filter_partition_pruning_timestamp1 -- 1 -- !runtime_filter_partition_pruning_timestamp2 -- 2 +-- !runtime_filter_partition_pruning_timestamp_in_null -- +1 + -- !runtime_filter_partition_pruning_boolean1 -- 3 -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_boolean_in_null -- +3 + -- !runtime_filter_partition_pruning_float1 -- 1 @@ -50,18 +68,27 @@ -- !runtime_filter_partition_pruning_float3 -- 1 +-- !runtime_filter_partition_pruning_float_in_null -- +1 + -- !runtime_filter_partition_pruning_timestamp_ntz1 -- 1 -- !runtime_filter_partition_pruning_timestamp_ntz2 -- 2 +-- !runtime_filter_partition_pruning_timestamp_ntz_in_null -- +1 + -- !runtime_filter_partition_pruning_binary1 -- 1 -- !runtime_filter_partition_pruning_binary2 -- 3 +-- !runtime_filter_partition_pruning_binary_in_null -- +1 + -- !runtime_filter_partition_pruning_decimal1 -- 2 @@ -71,6 +98,9 @@ -- !runtime_filter_partition_pruning_decimal3 -- 2 +-- !runtime_filter_partition_pruning_decimal_in_null -- +2 + -- !runtime_filter_partition_pruning_int1 -- 1 @@ -80,30 +110,45 @@ -- !runtime_filter_partition_pruning_int3 -- 1 +-- !runtime_filter_partition_pruning_int_in_null -- +1 + -- !runtime_filter_partition_pruning_string1 -- 2 -- !runtime_filter_partition_pruning_string2 -- 4 +-- !runtime_filter_partition_pruning_string_in_null -- +2 + -- !runtime_filter_partition_pruning_date1 -- 1 -- !runtime_filter_partition_pruning_date2 -- 3 +-- !runtime_filter_partition_pruning_date_in_null -- +1 + -- !runtime_filter_partition_pruning_timestamp1 -- 1 -- !runtime_filter_partition_pruning_timestamp2 -- 2 +-- !runtime_filter_partition_pruning_timestamp_in_null -- +1 + -- !runtime_filter_partition_pruning_boolean1 -- 3 -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_boolean_in_null -- +3 + -- !runtime_filter_partition_pruning_float1 -- 1 @@ -113,15 +158,24 @@ -- !runtime_filter_partition_pruning_float3 -- 1 +-- !runtime_filter_partition_pruning_float_in_null -- +1 + -- !runtime_filter_partition_pruning_timestamp_ntz1 -- 1 -- !runtime_filter_partition_pruning_timestamp_ntz2 -- 2 +-- !runtime_filter_partition_pruning_timestamp_ntz_in_null -- +1 + -- !runtime_filter_partition_pruning_binary1 -- 1 -- !runtime_filter_partition_pruning_binary2 -- 3 +-- !runtime_filter_partition_pruning_binary_in_null -- +1 + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index acd471479f862c..76b6cf5d9d45f1 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -8,6 +8,9 @@ -- !runtime_filter_partition_pruning_decimal3 -- 2 +-- !runtime_filter_partition_pruning_decimal_in_null -- +2 + -- !runtime_filter_partition_pruning_int1 -- 1 @@ -17,24 +20,36 @@ -- !runtime_filter_partition_pruning_int3 -- 1 +-- !runtime_filter_partition_pruning_int_in_null -- +1 + -- !runtime_filter_partition_pruning_string1 -- 2 -- !runtime_filter_partition_pruning_string2 -- 4 +-- !runtime_filter_partition_pruning_string_in_null -- +2 + -- !runtime_filter_partition_pruning_date1 -- 1 -- !runtime_filter_partition_pruning_date2 -- 3 +-- !runtime_filter_partition_pruning_date_in_null -- +1 + -- !runtime_filter_partition_pruning_timestamp1 -- 1 -- !runtime_filter_partition_pruning_timestamp2 -- 2 +-- !runtime_filter_partition_pruning_timestamp_in_null -- +1 + -- !runtime_filter_partition_pruning_bigint1 -- 1 @@ -44,12 +59,18 @@ -- !runtime_filter_partition_pruning_bigint3 -- 1 +-- !runtime_filter_partition_pruning_bigint_in_null -- +1 + -- !runtime_filter_partition_pruning_boolean1 -- 3 -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_boolean_in_null -- +3 + -- !runtime_filter_partition_pruning_float1 -- 1 @@ -59,6 +80,9 @@ -- !runtime_filter_partition_pruning_float3 -- 1 +-- !runtime_filter_partition_pruning_float_in_null -- +1 + -- !runtime_filter_partition_pruning_decimal1 -- 2 @@ -68,6 +92,9 @@ -- !runtime_filter_partition_pruning_decimal3 -- 2 +-- !runtime_filter_partition_pruning_decimal_in_null -- +2 + -- !runtime_filter_partition_pruning_int1 -- 1 @@ -77,24 +104,36 @@ -- !runtime_filter_partition_pruning_int3 -- 1 +-- !runtime_filter_partition_pruning_int_in_null -- +1 + -- !runtime_filter_partition_pruning_string1 -- 2 -- !runtime_filter_partition_pruning_string2 -- 4 +-- !runtime_filter_partition_pruning_string_in_null -- +2 + -- !runtime_filter_partition_pruning_date1 -- 1 -- !runtime_filter_partition_pruning_date2 -- 3 +-- !runtime_filter_partition_pruning_date_in_null -- +1 + -- !runtime_filter_partition_pruning_timestamp1 -- 1 -- !runtime_filter_partition_pruning_timestamp2 -- 2 +-- !runtime_filter_partition_pruning_timestamp_in_null -- +1 + -- !runtime_filter_partition_pruning_bigint1 -- 1 @@ -104,12 +143,18 @@ -- !runtime_filter_partition_pruning_bigint3 -- 1 +-- !runtime_filter_partition_pruning_bigint_in_null -- +1 + -- !runtime_filter_partition_pruning_boolean1 -- 3 -- !runtime_filter_partition_pruning_boolean2 -- 5 +-- !runtime_filter_partition_pruning_boolean_in_null -- +3 + -- !runtime_filter_partition_pruning_float1 -- 1 @@ -119,3 +164,6 @@ -- !runtime_filter_partition_pruning_float3 -- 1 +-- !runtime_filter_partition_pruning_float_in_null -- +1 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index 630a95f267fe68..47729fc2c5de9f 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -62,6 +62,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_decimal_in_null """ + select count(*) from decimal_partitioned where partition_key in + (select partition_key from decimal_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_int1 """ select count(*) from int_partitioned where partition_key = (select partition_key from int_partitioned @@ -80,6 +85,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_int_in_null """ + select count(*) from int_partitioned where partition_key in + (select partition_key from int_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_string1 """ select count(*) from string_partitioned where partition_key = (select partition_key from string_partitioned @@ -92,6 +102,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_string_in_null """ + select count(*) from string_partitioned where partition_key in + (select partition_key from string_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_date1 """ select count(*) from date_partitioned where partition_key = (select partition_key from date_partitioned @@ -104,6 +119,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_date_in_null """ + select count(*) from date_partitioned where partition_key in + (select partition_key from date_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_timestamp1 """ select count(*) from timestamp_partitioned where partition_key = (select partition_key from timestamp_partitioned @@ -116,6 +136,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_timestamp_in_null """ + select count(*) from timestamp_partitioned where partition_key in + (select partition_key from timestamp_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_boolean1 """ select count(*) from boolean_partitioned where partition_key = (select partition_key from boolean_partitioned @@ -127,6 +152,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern (select partition_key from boolean_partitioned group by partition_key having count(*) > 0); """ + qt_runtime_filter_partition_pruning_boolean_in_null """ + select count(*) from boolean_partitioned where partition_key in + (select partition_key from boolean_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_float1 """ select count(*) from float_partitioned where partition_key = (select partition_key from float_partitioned @@ -145,6 +175,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_float_in_null """ + select count(*) from float_partitioned where partition_key in + (select partition_key from float_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_timestamp_ntz1 """ select count(*) from timestamp_ntz_partitioned where partition_key = (select partition_key from timestamp_ntz_partitioned @@ -157,6 +192,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_timestamp_ntz_in_null """ + select count(*) from timestamp_ntz_partitioned where partition_key in + (select partition_key from timestamp_ntz_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_binary1 """ select count(*) from binary_partitioned where partition_key = (select partition_key from binary_partitioned @@ -169,6 +209,11 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_binary_in_null """ + select count(*) from binary_partitioned where partition_key in + (select partition_key from binary_partitioned + order by id desc limit 2); + """ } try { sql """ set time_zone = 'Asia/Shanghai'; """ diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy index 3e99892cd92c83..dc09444529ceed 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy @@ -57,6 +57,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_decimal_in_null """ + select count(*) from decimal_partitioned where partition_key in + (select partition_key from decimal_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_int1 """ select count(*) from int_partitioned where partition_key = (select partition_key from int_partitioned @@ -75,6 +80,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_int_in_null """ + select count(*) from int_partitioned where partition_key in + (select partition_key from int_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_string1 """ select count(*) from string_partitioned where partition_key = (select partition_key from string_partitioned @@ -87,6 +97,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_string_in_null """ + select count(*) from string_partitioned where partition_key in + (select partition_key from string_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_date1 """ select count(*) from date_partitioned where partition_key = (select partition_key from date_partitioned @@ -99,6 +114,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_date_in_null """ + select count(*) from date_partitioned where partition_key in + (select partition_key from date_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_timestamp1 """ select count(*) from timestamp_partitioned where partition_key = (select partition_key from timestamp_partitioned @@ -111,6 +131,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 2); """ + qt_runtime_filter_partition_pruning_timestamp_in_null """ + select count(*) from timestamp_partitioned where partition_key in + (select partition_key from timestamp_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_bigint1 """ select count(*) from bigint_partitioned where partition_key = (select partition_key from bigint_partitioned @@ -129,6 +154,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_bigint_in_null """ + select count(*) from bigint_partitioned where partition_key in + (select partition_key from bigint_partitioned + order by id desc limit 2); + """ qt_runtime_filter_partition_pruning_boolean1 """ select count(*) from boolean_partitioned where partition_key = (select partition_key from boolean_partitioned @@ -140,6 +170,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa (select partition_key from boolean_partitioned group by partition_key having count(*) > 0); """ + qt_runtime_filter_partition_pruning_boolean_in_null """ + select count(*) from boolean_partitioned where partition_key in + (select partition_key from boolean_partitioned + order by id desc limit 2); + """ // binary type as partition key will cause issues in paimon, so skipping these tests // qt_runtime_filter_partition_pruning_binary1 """ // select count(*) from binary_partitioned where partition_key = @@ -171,6 +206,11 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa group by partition_key having count(*) > 0 order by partition_key desc limit 1); """ + qt_runtime_filter_partition_pruning_float_in_null """ + select count(*) from float_partitioned where partition_key in + (select partition_key from float_partitioned + order by id desc limit 2); + """ } try { From 92ed4cd39c0f0d6c2f1c1c95456daec2870d3c28 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 4 Aug 2025 14:23:05 +0800 Subject: [PATCH 38/43] add columns_from_path_is_null --- gensrc/thrift/PlanNodes.thrift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index bbdc414cbd71ed..18736a0774b6e6 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -493,6 +493,8 @@ struct TFileRangeDesc { 12: optional string fs_name 13: optional TFileFormatType format_type; 14: optional i64 self_split_weight + // whether the value of columns_from_path is null + 15: optional list columns_from_path_is_null; } struct TSplitSource { From 4e58a459ca09bc0d7b2a862d0ed5209829a35112 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 4 Aug 2025 16:59:54 +0800 Subject: [PATCH 39/43] fix be --- be/src/vec/exec/scan/file_scanner.cpp | 31 +++++++++++++++++++++++---- be/src/vec/exec/scan/file_scanner.h | 1 + 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 3997dd7b5ae485..861b1572c9606c 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -253,13 +253,32 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al std::unordered_map partition_slot_id_to_column; for (auto const& partition_col_desc : _partition_col_descs) { const auto& [partition_value, partition_slot_desc] = partition_col_desc.second; - auto test_serde = partition_slot_desc->get_data_type_ptr()->get_serde(); - auto partition_value_column = partition_slot_desc->get_data_type_ptr()->create_column(); + auto data_type = partition_slot_desc->get_data_type_ptr(); + auto test_serde = data_type->get_serde(); + auto partition_value_column = data_type->create_column(); auto* col_ptr = static_cast(partition_value_column.get()); Slice slice(partition_value.data(), partition_value.size()); uint64_t num_deserialized = 0; - RETURN_IF_ERROR(test_serde->deserialize_column_from_fixed_json( - *col_ptr, slice, partition_value_column_size, &num_deserialized, {})); + DataTypeSerDe::FormatOptions options {}; + if (_partition_value_is_null.contains(partition_slot_desc->col_name())) { + // for iceberg/paimon table + test_serde = data_type->is_nullable() ? test_serde->get_nested_serdes()[0] : test_serde; + bool is_null = _partition_value_is_null[partition_slot_desc->col_name()]; + if (is_null) { + // If the partition value is null, insert default values into the ColumnNullable. + col_ptr->insert_many_defaults(partition_value_column_size); + } else { + // If the partition value is not null, we deserialize it normally. + RETURN_IF_ERROR(test_serde->deserialize_column_from_fixed_json( + *col_ptr, slice, partition_value_column_size, &num_deserialized, options)); + } + } else { + // for hive/hudi table, the null value is set as "\\N" + // TODO: this will be unified as iceberg/paimon table in the future + RETURN_IF_ERROR(test_serde->deserialize_column_from_fixed_json( + *col_ptr, slice, partition_value_column_size, &num_deserialized, options)); + } + partition_slot_id_to_column[partition_slot_desc->id()] = std::move(partition_value_column); } @@ -1482,6 +1501,10 @@ Status FileScanner::_generate_partition_columns() { } _partition_col_descs.emplace(slot_desc->col_name(), std::make_tuple(data, slot_desc)); + if (range.__isset.columns_from_path_is_null) { + _partition_value_is_null.emplace(slot_desc->col_name(), + range.columns_from_path_is_null[it->second]); + } } } } diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h index 3556897b6274a3..290d2ff2f2ea28 100644 --- a/be/src/vec/exec/scan/file_scanner.h +++ b/be/src/vec/exec/scan/file_scanner.h @@ -192,6 +192,7 @@ class FileScanner : public Scanner { bool _fill_partition_from_path = true; std::unordered_map> _partition_col_descs; + std::unordered_map _partition_value_is_null; std::unordered_map _missing_col_descs; // idx of skip_bitmap_col in _input_tuple_desc From 82acb703da06a7d5a5eb0fc1b3f464a0811c86ba Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 4 Aug 2025 17:09:21 +0800 Subject: [PATCH 40/43] fix fe --- .../doris/datasource/iceberg/IcebergUtils.java | 11 +++++------ .../iceberg/source/IcebergScanNode.java | 15 +++++++++------ .../doris/datasource/paimon/PaimonUtil.java | 13 ++++++------- .../datasource/paimon/source/PaimonScanNode.java | 15 +++++++++------ 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index aba7d0da928c1e..90155157a3e610 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -49,7 +49,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CacheException; @@ -637,13 +636,13 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type case UUID: case DECIMAL: if (value == null) { - return FeConstants.null_string; + return null; } return value.toString(); case FIXED: case BINARY: if (value == null) { - return FeConstants.null_string; + return null; } // Fixed and binary types are stored as ByteBuffer ByteBuffer buffer = (ByteBuffer) value; @@ -652,14 +651,14 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type return new String(res, StandardCharsets.UTF_8); case DATE: if (value == null) { - return FeConstants.null_string; + return null; } // Iceberg date is stored as days since epoch (1970-01-01) LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME: if (value == null) { - return FeConstants.null_string; + return null; } // Iceberg time is stored as microseconds since midnight long micros = (Long) value; @@ -667,7 +666,7 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP: if (value == null) { - return FeConstants.null_string; + return null; } // Iceberg timestamp is stored as microseconds since epoch // (1970-01-01T00:00:00) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 1d7ce7fcde4be5..badac3ba6c058c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -204,14 +204,17 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli tableFormatFileDesc.setIcebergParams(fileDesc); Map partitionValues = icebergSplit.getIcebergPartitionValues(); if (partitionValues != null) { - List formPathKeys = new ArrayList<>(); - List formPathValues = new ArrayList<>(); + List fromPathKeys = new ArrayList<>(); + List fromPathValues = new ArrayList<>(); + List fromPathIsNull = new ArrayList<>(); for (Map.Entry entry : partitionValues.entrySet()) { - formPathKeys.add(entry.getKey()); - formPathValues.add(entry.getValue()); + fromPathKeys.add(entry.getKey()); + fromPathValues.add(entry.getValue() != null ? entry.getValue() : ""); + fromPathIsNull.add(entry.getValue() == null); } - rangeDesc.setColumnsFromPathKeys(formPathKeys); - rangeDesc.setColumnsFromPath(formPathValues); + rangeDesc.setColumnsFromPathKeys(fromPathKeys); + rangeDesc.setColumnsFromPath(fromPathValues); + rangeDesc.setColumnsFromPathIsNull(fromPathIsNull); } rangeDesc.setTableFormatParams(tableFormatFileDesc); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 319f71fb775711..4d6fd91f23adbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -25,7 +25,6 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.hive.HiveUtil; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; @@ -422,25 +421,25 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t case VARCHAR: case CHAR: if (value == null) { - return FeConstants.null_string; + return null; } return value.toString(); case BINARY: case VARBINARY: if (value == null) { - return FeConstants.null_string; + return null; } return new String((byte[]) value, StandardCharsets.UTF_8); case DATE: if (value == null) { - return FeConstants.null_string; + return null; } // Paimon date is stored as days since epoch LocalDate date = LocalDate.ofEpochDay((Integer) value); return date.format(DateTimeFormatter.ISO_LOCAL_DATE); case TIME_WITHOUT_TIME_ZONE: if (value == null) { - return FeConstants.null_string; + return null; } // Paimon time is stored as microseconds since midnight in utc long micros = (Long) value; @@ -448,13 +447,13 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t return time.format(DateTimeFormatter.ISO_LOCAL_TIME); case TIMESTAMP_WITHOUT_TIME_ZONE: if (value == null) { - return FeConstants.null_string; + return null; } // Paimon timestamp is stored as Timestamp type in utc return ((Timestamp) value).toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); case TIMESTAMP_WITH_LOCAL_TIME_ZONE: if (value == null) { - return FeConstants.null_string; + return null; } // Paimon timestamp with local time zone is stored as Timestamp type in utc Timestamp timestamp = (Timestamp) value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 84d98e48c9a122..ff125f0d501f9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -243,14 +243,17 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) tableFormatFileDesc.setPaimonParams(fileDesc); Map partitionValues = paimonSplit.getPaimonPartitionValues(); if (partitionValues != null) { - List formPathKeys = new ArrayList<>(); - List formPathValues = new ArrayList<>(); + List fromPathKeys = new ArrayList<>(); + List fromPathValues = new ArrayList<>(); + List fromPathIsNull = new ArrayList<>(); for (Map.Entry entry : partitionValues.entrySet()) { - formPathKeys.add(entry.getKey()); - formPathValues.add(entry.getValue()); + fromPathKeys.add(entry.getKey()); + fromPathValues.add(entry.getValue() != null ? entry.getValue() : ""); + fromPathIsNull.add(entry.getValue() == null); } - rangeDesc.setColumnsFromPathKeys(formPathKeys); - rangeDesc.setColumnsFromPath(formPathValues); + rangeDesc.setColumnsFromPathKeys(fromPathKeys); + rangeDesc.setColumnsFromPath(fromPathValues); + rangeDesc.setColumnsFromPathIsNull(fromPathIsNull); } rangeDesc.setTableFormatParams(tableFormatFileDesc); } From 9aa46a56c4983b5b919084ab010a6e9305022737 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 4 Aug 2025 20:38:41 +0800 Subject: [PATCH 41/43] fix and add regression --- be/src/vec/exec/scan/file_scanner.cpp | 25 +++++++++---------- .../iceberg/run18.sql | 17 ++++++++++++- .../paimon/run06.sql | 17 ++++++++++++- .../doris/datasource/paimon/PaimonUtil.java | 9 +++++-- ...eberg_runtime_filter_partition_pruning.out | 24 ++++++++++++++++++ ...aimon_runtime_filter_partition_pruning.out | 24 ++++++++++++++++++ ...rg_runtime_filter_partition_pruning.groovy | 12 +++++++++ ...on_runtime_filter_partition_pruning.groovy | 12 +++++++++ 8 files changed, 123 insertions(+), 17 deletions(-) diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp index 861b1572c9606c..a21243f3d1dfd5 100644 --- a/be/src/vec/exec/scan/file_scanner.cpp +++ b/be/src/vec/exec/scan/file_scanner.cpp @@ -262,15 +262,18 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al DataTypeSerDe::FormatOptions options {}; if (_partition_value_is_null.contains(partition_slot_desc->col_name())) { // for iceberg/paimon table - test_serde = data_type->is_nullable() ? test_serde->get_nested_serdes()[0] : test_serde; - bool is_null = _partition_value_is_null[partition_slot_desc->col_name()]; - if (is_null) { - // If the partition value is null, insert default values into the ColumnNullable. - col_ptr->insert_many_defaults(partition_value_column_size); + // NOTICE: column is always be nullable for iceberg/paimon table now + DCHECK(data_type->is_nullable()); + test_serde = test_serde->get_nested_serdes()[0]; + auto* null_column = assert_cast(col_ptr); + if (_partition_value_is_null[partition_slot_desc->col_name()]) { + null_column->insert_many_defaults(partition_value_column_size); } else { - // If the partition value is not null, we deserialize it normally. + // If the partition value is not null, we set null map to 0 and deserialize it normally. + null_column->get_null_map_column().insert_many_vals(0, partition_value_column_size); RETURN_IF_ERROR(test_serde->deserialize_column_from_fixed_json( - *col_ptr, slice, partition_value_column_size, &num_deserialized, options)); + null_column->get_nested_column(), slice, partition_value_column_size, + &num_deserialized, options)); } } else { // for hive/hudi table, the null value is set as "\\N" @@ -1484,6 +1487,7 @@ Status FileScanner::read_lines_from_range(const TFileRangeDesc& range, Status FileScanner::_generate_partition_columns() { _partition_col_descs.clear(); + _partition_value_is_null.clear(); const TFileRangeDesc& range = _current_range; if (range.__isset.columns_from_path && !_partition_slot_descs.empty()) { for (const auto& slot_desc : _partition_slot_descs) { @@ -1494,13 +1498,8 @@ Status FileScanner::_generate_partition_columns() { slot_desc->id()); } const std::string& column_from_path = range.columns_from_path[it->second]; - const char* data = column_from_path.c_str(); - size_t size = column_from_path.size(); - if (size == 4 && memcmp(data, "null", 4) == 0) { - data = const_cast("\\N"); - } _partition_col_descs.emplace(slot_desc->col_name(), - std::make_tuple(data, slot_desc)); + std::make_tuple(column_from_path, slot_desc)); if (range.__isset.columns_from_path_is_null) { _partition_value_is_null.emplace(slot_desc->col_name(), range.columns_from_path_is_null[it->second]); diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql index 77bfad8fdb0587..84c00e12781f71 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run18.sql @@ -232,4 +232,19 @@ VALUES ( 6, 'Null Binary', NULL - ); \ No newline at end of file + ); + +-- Partition by string type with null values +CREATE TABLE null_str_partition_table ( + id BIGINT, + category STRING, + value DOUBLE +) USING iceberg PARTITIONED BY (category); + +INSERT INTO + null_str_partition_table +VALUES (1, NULL, 100.0), + (2, 'NULL', 200.0), + (3, '\\N', 300.0), + (4, 'null', 400.0), + (5, 'A', 500.0); \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql index bf5e436e9d8ca9..b1352c623c593d 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -212,4 +212,19 @@ VALUES (1, 'Float Data 1', 1.5), (3, 'Float Data 3', 2.5), (4, 'Float Data 4', 2.5), (5, 'Float Data 5', 3.5), - (6, 'Null Float', NULL); \ No newline at end of file + (6, 'Null Float', NULL); + +-- Partition by string type with null values +CREATE TABLE null_str_partition_table ( + id BIGINT, + category STRING, + value DOUBLE +) PARTITIONED BY (category); + +INSERT INTO + null_str_partition_table +VALUES (1, NULL, 100.0), + (2, 'NULL', 200.0), + (3, '\\N', 300.0), + (4, 'null', 400.0), + (5, 'A', 500.0); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 4d6fd91f23adbb..4c147c8456c97c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -84,6 +84,7 @@ public class PaimonUtil { private static final Logger LOG = LogManager.getLogger(PaimonUtil.class); private static final Base64.Encoder BASE64_ENCODER = java.util.Base64.getUrlEncoder().withoutPadding(); + private static final String DEFAULT_PARTITION_VALUE = "__DEFAULT_PARTITION__"; public static List read( Table table, @Nullable int[] projection, @Nullable Predicate predicate, @@ -136,8 +137,12 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo StringBuilder sb = new StringBuilder(); for (Map.Entry entry : spec.entrySet()) { sb.append(entry.getKey()).append("="); - // Paimon stores DATE type as days since 1970-01-01 (epoch), so we convert the integer to a date string. - if (columnNameToType.getOrDefault(entry.getKey(), Type.NULL).isDateV2()) { + if (entry.getValue().equals(DEFAULT_PARTITION_VALUE)) { + // Paimon uses "__DEFAULT_PARTITION__" to represent the default partition. + sb.append(entry.getValue()).append("/"); + } else if (columnNameToType.getOrDefault(entry.getKey(), Type.NULL).isDateV2()) { + // Paimon stores DATE type as days since 1970-01-01 (epoch), so we convert the + // integer to a date string. sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/"); } else { sb.append(entry.getValue()).append("/"); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index 67d91cb79ee066..3198e40353be8b 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -89,6 +89,18 @@ -- !runtime_filter_partition_pruning_binary_in_null -- 1 +-- !null_partition_1 -- +4 null 400.0 + +-- !null_partition_2 -- +2 NULL 200.0 + +-- !null_partition_3 -- +3 \\N 300.0 + +-- !null_partition_4 -- +1 \N 100.0 + -- !runtime_filter_partition_pruning_decimal1 -- 2 @@ -179,3 +191,15 @@ -- !runtime_filter_partition_pruning_binary_in_null -- 1 +-- !null_partition_1 -- +4 null 400.0 + +-- !null_partition_2 -- +2 NULL 200.0 + +-- !null_partition_3 -- +3 \\N 300.0 + +-- !null_partition_4 -- +1 \N 100.0 + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index 76b6cf5d9d45f1..6d57272155b3bf 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -83,6 +83,18 @@ -- !runtime_filter_partition_pruning_float_in_null -- 1 +-- !null_partition_1 -- +4 null 400.0 + +-- !null_partition_2 -- +2 NULL 200.0 + +-- !null_partition_3 -- +3 \\N 300.0 + +-- !null_partition_4 -- +1 \N 100.0 + -- !runtime_filter_partition_pruning_decimal1 -- 2 @@ -167,3 +179,15 @@ -- !runtime_filter_partition_pruning_float_in_null -- 1 +-- !null_partition_1 -- +4 null 400.0 + +-- !null_partition_2 -- +2 NULL 200.0 + +-- !null_partition_3 -- +3 \\N 300.0 + +-- !null_partition_4 -- +1 \N 100.0 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index 47729fc2c5de9f..f69fe71e2c4cf6 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -214,6 +214,18 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern (select partition_key from binary_partitioned order by id desc limit 2); """ + qt_null_partition_1 """ + select * from null_str_partition_table where category = 'null'; + """ + qt_null_partition_2 """ + select * from null_str_partition_table where category = 'NULL'; + """ + qt_null_partition_3 """ + select * from null_str_partition_table where category = '\\\\N'; + """ + qt_null_partition_4 """ + select * from null_str_partition_table where category is null; + """ } try { sql """ set time_zone = 'Asia/Shanghai'; """ diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy index dc09444529ceed..a0bfca1693226a 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy @@ -211,6 +211,18 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa (select partition_key from float_partitioned order by id desc limit 2); """ + qt_null_partition_1 """ + select * from null_str_partition_table where category = 'null'; + """ + qt_null_partition_2 """ + select * from null_str_partition_table where category = 'NULL'; + """ + qt_null_partition_3 """ + select * from null_str_partition_table where category = '\\\\N'; + """ + qt_null_partition_4 """ + select * from null_str_partition_table where category is null; + """ } try { From f0b595b665ddc5d04e48e5bc8b5c71af2e25f163 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 5 Aug 2025 10:28:01 +0800 Subject: [PATCH 42/43] fix regression --- .../iceberg/test_iceberg_runtime_filter_partition_pruning.out | 4 ++-- .../paimon/test_paimon_runtime_filter_partition_pruning.out | 4 ++-- .../test_iceberg_runtime_filter_partition_pruning.groovy | 2 +- .../test_paimon_runtime_filter_partition_pruning.groovy | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out index 3198e40353be8b..530973b08276ea 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.out @@ -96,7 +96,7 @@ 2 NULL 200.0 -- !null_partition_3 -- -3 \\N 300.0 +1 -- !null_partition_4 -- 1 \N 100.0 @@ -198,7 +198,7 @@ 2 NULL 200.0 -- !null_partition_3 -- -3 \\N 300.0 +1 -- !null_partition_4 -- 1 \N 100.0 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index 6d57272155b3bf..8c26e76ff63119 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -90,7 +90,7 @@ 2 NULL 200.0 -- !null_partition_3 -- -3 \\N 300.0 +1 -- !null_partition_4 -- 1 \N 100.0 @@ -186,7 +186,7 @@ 2 NULL 200.0 -- !null_partition_3 -- -3 \\N 300.0 +1 -- !null_partition_4 -- 1 \N 100.0 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy index f69fe71e2c4cf6..442a8a4c1218a1 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_runtime_filter_partition_pruning.groovy @@ -221,7 +221,7 @@ suite("test_iceberg_runtime_filter_partition_pruning", "p0,external,doris,extern select * from null_str_partition_table where category = 'NULL'; """ qt_null_partition_3 """ - select * from null_str_partition_table where category = '\\\\N'; + select count(*) from null_str_partition_table where category = '\\\\N'; """ qt_null_partition_4 """ select * from null_str_partition_table where category is null; diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy index a0bfca1693226a..f7a666d2c8328f 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.groovy @@ -218,7 +218,7 @@ suite("test_paimon_runtime_filter_partition_pruning", "p0,external,doris,externa select * from null_str_partition_table where category = 'NULL'; """ qt_null_partition_3 """ - select * from null_str_partition_table where category = '\\\\N'; + select count(*) from null_str_partition_table where category = '\\\\N'; """ qt_null_partition_4 """ select * from null_str_partition_table where category is null; From 62ccb786cf9acad4df415dfc99f11fba29bdec1a Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 5 Aug 2025 17:07:29 +0800 Subject: [PATCH 43/43] fix regression --- .../scripts/create_preinstalled_scripts/paimon/run06.sql | 5 +++-- .../org/apache/doris/datasource/paimon/PaimonUtil.java | 9 ++------- .../test_paimon_runtime_filter_partition_pruning.out | 4 ++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql index b1352c623c593d..eb60255a08e965 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run06.sql @@ -25,8 +25,9 @@ VALUES (1, 'Alice', DATE '2024-01-01'), DATE '2024-02-01' ), (4, 'David', DATE '2024-02-01'), - (5, 'Eve', DATE '2024-03-01'), - (6, 'Null Date', NULL); + (5, 'Eve', DATE '2024-03-01'); + -- TODO: add this after fix paimon null date partition issue + -- (6, 'Null Date', NULL); -- Partition by timestamp type CREATE TABLE timestamp_partitioned ( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 4c147c8456c97c..4d6fd91f23adbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -84,7 +84,6 @@ public class PaimonUtil { private static final Logger LOG = LogManager.getLogger(PaimonUtil.class); private static final Base64.Encoder BASE64_ENCODER = java.util.Base64.getUrlEncoder().withoutPadding(); - private static final String DEFAULT_PARTITION_VALUE = "__DEFAULT_PARTITION__"; public static List read( Table table, @Nullable int[] projection, @Nullable Predicate predicate, @@ -137,12 +136,8 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo StringBuilder sb = new StringBuilder(); for (Map.Entry entry : spec.entrySet()) { sb.append(entry.getKey()).append("="); - if (entry.getValue().equals(DEFAULT_PARTITION_VALUE)) { - // Paimon uses "__DEFAULT_PARTITION__" to represent the default partition. - sb.append(entry.getValue()).append("/"); - } else if (columnNameToType.getOrDefault(entry.getKey(), Type.NULL).isDateV2()) { - // Paimon stores DATE type as days since 1970-01-01 (epoch), so we convert the - // integer to a date string. + // Paimon stores DATE type as days since 1970-01-01 (epoch), so we convert the integer to a date string. + if (columnNameToType.getOrDefault(entry.getKey(), Type.NULL).isDateV2()) { sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/"); } else { sb.append(entry.getValue()).append("/"); diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out index 8c26e76ff63119..738d40fabe0498 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_runtime_filter_partition_pruning.out @@ -39,7 +39,7 @@ 3 -- !runtime_filter_partition_pruning_date_in_null -- -1 +3 -- !runtime_filter_partition_pruning_timestamp1 -- 1 @@ -135,7 +135,7 @@ 3 -- !runtime_filter_partition_pruning_date_in_null -- -1 +3 -- !runtime_filter_partition_pruning_timestamp1 -- 1