Revert "[improve](mow) merge and remove old version of delete bitmap …

…when cumulative compaction is done (apache#186)" (apache#188)

This reverts commit 8a75deb.

be/src/olap/compaction.cpp

@@ -1202,34 +1202,6 @@ void Compaction::_load_segment_to_cache() {
     }
 }
 
-void Compaction::agg_and_remove_old_version_delete_bitmap(
-        std::vector<RowsetSharedPtr>& pre_rowsets,
-        std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>&
-                to_remove_vec,
-        DeleteBitmapPtr& new_delete_bitmap) {
-    // agg previously rowset old version delete bitmap
-    auto pre_max_version = _output_rowset->version().second;
-    new_delete_bitmap = std::make_shared<DeleteBitmap>(_tablet->tablet_meta()->tablet_id());
-    for (auto& rowset : pre_rowsets) {
-        if (rowset->rowset_meta()->total_disk_size() == 0) {
-            continue;
-        }
-        for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
-            rowset->rowset_id().to_string();
-            DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
-            DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version};
-            DeleteBitmap::BitmapKey before_end {rowset->rowset_id(), seg_id, pre_max_version - 1};
-            auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
-                    {rowset->rowset_id(), seg_id, pre_max_version});
-            to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, before_end));
-            if (d->isEmpty()) {
-                continue;
-            }
-            new_delete_bitmap->set(end, *d);
-        }
-    }
-}
-
 #ifdef BE_TEST
 void Compaction::set_input_rowset(const std::vector<RowsetSharedPtr>& rowsets) {
     _input_rowsets = rowsets;

be/src/olap/compaction.h

@@ -102,12 +102,6 @@ class Compaction {
         return _allow_delete_in_cumu_compaction;
     }
 
-    void agg_and_remove_old_version_delete_bitmap(
-            std::vector<RowsetSharedPtr>& pre_rowsets,
-            std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>&
-                    to_remove_vec,
-            DeleteBitmapPtr& new_delete_bitmap);
-
 private:
     bool _check_if_includes_input_rowsets(const RowsetIdUnorderedSet& commit_rowset_ids_set) const;
     void _load_segment_to_cache();

be/src/olap/cumulative_compaction.cpp

@@ -97,11 +97,6 @@ Status CumulativeCompaction::execute_compact_impl() {
     DorisMetrics::instance()->cumulative_compaction_deltas_total->increment(_input_rowsets.size());
     DorisMetrics::instance()->cumulative_compaction_bytes_total->increment(_input_rowsets_size);
 
-    if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
-        _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) {
-        _process_old_version_delete_bitmap();
-    }
-
     return Status::OK();
 }
 
@@ -193,34 +188,4 @@ Status CumulativeCompaction::pick_rowsets_to_compact() {
     return Status::OK();
 }
 
-void CumulativeCompaction::_process_old_version_delete_bitmap() {
-    std::vector<RowsetSharedPtr> pre_rowsets {};
-    for (const auto& it : _tablet->rowset_map()) {
-        if (it.first.second < _input_rowsets.front()->start_version()) {
-            pre_rowsets.emplace_back(it.second);
-        }
-    }
-    std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
-    if (!pre_rowsets.empty()) {
-        std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>
-                to_remove_vec;
-        DeleteBitmapPtr new_delete_bitmap = nullptr;
-        agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap);
-        if (!new_delete_bitmap->empty()) {
-            // store agg delete bitmap
-            Version version(_input_rowsets.front()->start_version(),
-                            _input_rowsets.back()->end_version());
-            for (auto it = new_delete_bitmap->delete_bitmap.begin();
-                 it != new_delete_bitmap->delete_bitmap.end(); it++) {
-                _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second);
-            }
-            _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(),
-                                                                        to_remove_vec);
-            DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", {
-                static_cast<Tablet*>(_tablet.get())->delete_expired_stale_rowset();
-            });
-        }
-    }
-}
-
 } // namespace doris

be/src/olap/cumulative_compaction.h

@@ -49,8 +49,6 @@ class CumulativeCompaction : public Compaction {
 private:
     Version _last_delete_version {-1, -1};
 
-    void _process_old_version_delete_bitmap();
-
     DISALLOW_COPY_AND_ASSIGN(CumulativeCompaction);
 };
 

be/src/olap/rowset/beta_rowset_writer.cpp

@@ -192,8 +192,7 @@ Status BetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
     LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id()
               << ", rowset_ids: " << _context.mow_context->rowset_ids.size()
               << ", cur max_version: " << _context.mow_context->max_version
-              << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: "
-              << _context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count()
+              << ", transaction_id: " << _context.mow_context->txn_id
               << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows;
     return Status::OK();
 }

be/src/olap/tablet.cpp

@@ -884,13 +884,10 @@ void Tablet::delete_expired_stale_rowset() {
         auto old_meta_size = _tablet_meta->all_stale_rs_metas().size();
 
         // do delete operation
-        std::vector<std::string> version_to_delete;
         auto to_delete_iter = stale_version_path_map.begin();
         while (to_delete_iter != stale_version_path_map.end()) {
             std::vector<TimestampedVersionSharedPtr>& to_delete_version =
                     to_delete_iter->second->timestamped_versions();
-            int start_version = -1;
-            int end_version = -1;
             for (auto& timestampedVersion : to_delete_version) {
                 auto it = _stale_rs_version_map.find(timestampedVersion->version());
                 if (it != _stale_rs_version_map.end()) {
@@ -909,17 +906,10 @@ void Tablet::delete_expired_stale_rowset() {
                                  << timestampedVersion->version().second
                                  << "] not find in stale rs version map";
                 }
-                if (start_version < 0) {
-                    start_version = timestampedVersion->version().first;
-                }
-                end_version = timestampedVersion->version().second;
                 _delete_stale_rowset_by_version(timestampedVersion->version());
             }
-            Version version(start_version, end_version);
-            version_to_delete.emplace_back(version.to_string());
             to_delete_iter++;
         }
-        _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete);
 
         bool reconstructed = _reconstruct_version_tracker_if_necessary();
 

be/src/olap/tablet_meta.cpp

@@ -1048,7 +1048,6 @@ bool DeleteBitmap::empty() const {
 }
 
 uint64_t DeleteBitmap::cardinality() const {
-    std::shared_lock l(lock);
     uint64_t res = 0;
     for (auto entry : delete_bitmap) {
         res += entry.second.cardinality();
@@ -1123,44 +1122,6 @@ void DeleteBitmap::merge(const DeleteBitmap& other) {
     }
 }
 
-size_t DeleteBitmap::get_size() const {
-    std::shared_lock l(lock);
-    size_t charge = 0;
-    for (auto& [k, v] : delete_bitmap) {
-        charge += v.getSizeInBytes();
-    }
-    return charge;
-}
-
-void DeleteBitmap::add_to_remove_queue(
-        const std::string& version_str,
-        const std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>&
-                vector) {
-    std::shared_lock l(stale_delete_bitmap_lock);
-    _stale_delete_bitmap.emplace(version_str, vector);
-}
-
-void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector<std::string>& vector) {
-    std::shared_lock l(stale_delete_bitmap_lock);
-    for (auto& version_str : vector) {
-        auto it = _stale_delete_bitmap.find(version_str);
-        if (it != _stale_delete_bitmap.end()) {
-            auto delete_bitmap_vector = it->second;
-            for (auto& delete_bitmap_tuple : it->second) {
-                auto start_bmk = std::get<1>(delete_bitmap_tuple);
-                auto end_bmk = std::get<2>(delete_bitmap_tuple);
-                remove(start_bmk, end_bmk);
-            }
-            _stale_delete_bitmap.erase(version_str);
-        }
-    }
-}
-
-uint64_t DeleteBitmap::get_delete_bitmap_count() {
-    std::shared_lock l(lock);
-    return delete_bitmap.size();
-}
-
 // We cannot just copy the underlying memory to construct a string
 // due to equivalent objects may have different padding bytes.
 // Reading padding bytes is undefined behavior, neither copy nor

be/src/olap/tablet_meta.h

@@ -352,7 +352,6 @@ class TabletMeta {
 class DeleteBitmap {
 public:
     mutable std::shared_mutex lock;
-    mutable std::shared_mutex stale_delete_bitmap_lock;
     using SegmentId = uint32_t;
     using Version = uint64_t;
     using BitmapKey = std::tuple<RowsetId, SegmentId, Version>;
@@ -432,12 +431,6 @@ class DeleteBitmap {
      */
     uint64_t cardinality() const;
 
-    /**
-     * return the total size of the Delete Bitmap(after serialized)
-     */
-
-    size_t get_size() const;
-
     /**
      * Sets the bitmap of specific segment, it's may be insertion or replacement
      *
@@ -506,13 +499,6 @@ class DeleteBitmap {
      */
     std::shared_ptr<roaring::Roaring> get_agg(const BitmapKey& bmk) const;
 
-    void add_to_remove_queue(const std::string& version_str,
-                             const std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey,
-                                                          DeleteBitmap::BitmapKey>>& vector);
-    void remove_stale_delete_bitmap_from_queue(const std::vector<std::string>& vector);
-
-    uint64_t get_delete_bitmap_count();
-
     class AggCachePolicy : public LRUCachePolicyTrackingManual {
     public:
         AggCachePolicy(size_t capacity)
@@ -547,10 +533,6 @@ class DeleteBitmap {
 private:
     mutable std::shared_ptr<AggCache> _agg_cache;
     int64_t _tablet_id;
-    // <version, <tablet_id, BitmapKeyStart, BitmapKeyEnd>>
-    std::map<std::string,
-             std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey, DeleteBitmap::BitmapKey>>>
-            _stale_delete_bitmap;
 };
 
 inline TabletUid TabletMeta::tablet_uid() const {

be/src/service/http_service.cpp

@@ -35,7 +35,6 @@
 #include "http/action/compaction_action.h"
 #include "http/action/config_action.h"
 #include "http/action/debug_point_action.h"
-#include "http/action/delete_bitmap_action.h"
 #include "http/action/download_action.h"
 #include "http/action/download_binlog_action.h"
 #include "http/action/file_cache_action.h"
@@ -277,12 +276,6 @@ Status HttpService::start() {
     _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status",
                                       run_status_compaction_action);
 
-    DeleteBitmapAction* count_delete_bitmap_action =
-            _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_INFO, _env,
-                                             TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
-    _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count",
-                                      count_delete_bitmap_action);
-
     ConfigAction* update_config_action =
             _pool.add(new ConfigAction(ConfigActionType::UPDATE_CONFIG));
     _ev_http_server->register_handler(HttpMethod::POST, "/api/update_config", update_config_action);