From 13da8f539613f803de4ddeffbbb3403469ad6fba Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 18 Jun 2025 16:28:49 +0800 Subject: [PATCH 1/3] full compaction prune delete sign data --- be/src/olap/tablet_reader.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index cc8daddb2c91ca..53943ef4a57211 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -650,7 +650,8 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates. // If delete design is applied on cumu compaction, it will lose effect when doing base compaction. // `_delete_sign_available` indicates the condition where we could apply delete signs to data. - _delete_sign_available = ((read_params.reader_type == ReaderType::READER_BASE_COMPACTION && + _delete_sign_available = (((read_params.reader_type == ReaderType::READER_BASE_COMPACTION || + read_params.reader_type == ReaderType::READER_FULL_COMPACTION) && config::enable_prune_delete_sign_when_base_compaction) || read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || read_params.reader_type == ReaderType::READER_CHECKSUM); From a1df4ba0b231e2262780dda3c33a8d134b03ad4d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 18 Jun 2025 16:38:48 +0800 Subject: [PATCH 2/3] check duplicate key for base compaction when enable_prune_delete_sign_when_base_compaction==false --- be/src/cloud/cloud_tablet.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 07fd356c541f50..1c046eeac8fe7c 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -1023,7 +1023,9 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( config::enable_mow_compaction_correctness_check_core || config::enable_mow_compaction_correctness_check_fail) && !allow_delete_in_cumu_compaction && - compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION) { + (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION || + !config::enable_prune_delete_sign_when_base_compaction)) { + // also check duplicate key for base compaction when config::enable_prune_delete_sign_when_base_compaction==false missed_rows = std::make_unique(); LOG(INFO) << "RowLocation Set inited succ for tablet:" << tablet_id(); } @@ -1045,7 +1047,8 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( if (missed_rows) { missed_rows_size = missed_rows->size(); if (!allow_delete_in_cumu_compaction) { - if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION && + if ((compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION || + !config::enable_prune_delete_sign_when_base_compaction) && tablet_state() == TABLET_RUNNING) { if (merged_rows + filtered_rows >= 0 && merged_rows + filtered_rows != missed_rows_size) { From 050dd909d610ccd396c82791cb10001d170f62a9 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 18 Jun 2025 19:28:29 +0800 Subject: [PATCH 3/3] fix case --- regression-test/data/compaction/test_full_compaction.out | 1 - .../data/compaction/test_full_compaction_by_table_id.out | 1 - 2 files changed, 2 deletions(-) diff --git a/regression-test/data/compaction/test_full_compaction.out b/regression-test/data/compaction/test_full_compaction.out index aaab47b4c3678b..91d01a244de892 100644 --- a/regression-test/data/compaction/test_full_compaction.out +++ b/regression-test/data/compaction/test_full_compaction.out @@ -39,7 +39,6 @@ -- !select_final -- 1 100 2 200 -3 0 -- !select_final2 -- 1 100 diff --git a/regression-test/data/compaction/test_full_compaction_by_table_id.out b/regression-test/data/compaction/test_full_compaction_by_table_id.out index b25fdad93145cc..118191b6cc8464 100644 --- a/regression-test/data/compaction/test_full_compaction_by_table_id.out +++ b/regression-test/data/compaction/test_full_compaction_by_table_id.out @@ -39,5 +39,4 @@ -- !select_final -- 1 100 2 200 -3 0