modify key

be/src/olap/compaction.cpp

@@ -49,6 +49,7 @@
 #include "olap/storage_policy.h"
 #include "olap/tablet.h"
 #include "olap/tablet_meta.h"
+#include "olap/tablet_meta_manager.h"
 #include "olap/task/engine_checksum_task.h"
 #include "olap/utils.h"
 #include "runtime/memory/mem_tracker_limiter.h"
@@ -623,10 +624,20 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) {
  RETURN_IF_ERROR(_tablet->modify_rowsets(output_rowsets, _input_rowsets, true));
  }
 
+ int64_t cur_max_version = 0;
  {
  std::shared_lock rlock(_tablet->get_header_lock());
+ cur_max_version = _tablet->max_version_unlocked().second;
  _tablet->save_meta();
  }
+ if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+ _tablet->enable_unique_key_merge_on_write()) {
+ auto st = TabletMetaManager::remove_old_version_delete_bitmap(
+ _tablet->data_dir(), _tablet->tablet_id(), cur_max_version);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to remove old version delete bitmap, st: " << st;
+ }
+ }
  return Status::OK();
 }
 

be/src/olap/data_dir.cpp

@@ -44,6 +44,7 @@
 #include "io/fs/local_file_system.h"
 #include "io/fs/path.h"
 #include "io/fs/remote_file_system.h"
+#include "olap/olap_common.h"
 #include "olap/olap_define.h"
 #include "olap/olap_meta.h"
 #include "olap/rowset/beta_rowset.h"
@@ -540,19 +541,41 @@ Status DataDir::load() {
  }
  }
 
- auto load_delete_bitmap_func = [this](int64_t tablet_id, RowsetId rowset_id, int64_t segment_id,
- int64_t version, const string& val) {
+ auto load_delete_bitmap_func = [this](int64_t tablet_id, int64_t version, const string& val) {
  TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id);
  if (!tablet) {
  return true;
  }
  const std::vector<RowsetMetaSharedPtr>& all_rowsets = tablet->tablet_meta()->all_rs_metas();
+ RowsetIdUnorderedSet rowset_ids;
  for (auto& rowset_meta : all_rowsets) {
+ rowset_ids.insert(rowset_meta->rowset_id());
+ }
+
+ DeleteBitmapPB delete_bitmap_pb;
+ delete_bitmap_pb.ParseFromString(val);
+ int rst_ids_size = delete_bitmap_pb.rowset_ids_size();
+ int seg_ids_size = delete_bitmap_pb.segment_ids_size();
+ int seg_maps_size = delete_bitmap_pb.segment_delete_bitmaps_size();
+ CHECK(rst_ids_size == seg_ids_size && seg_ids_size == seg_maps_size);
+
+ for (size_t i = 0; i < rst_ids_size; ++i) {
+ RowsetId rst_id;
+ rst_id.init(delete_bitmap_pb.rowset_ids(i));
  // only process the rowset in _rs_metas
- if (rowset_meta->rowset_id() == rowset_id) {
- tablet->tablet_meta()->delete_bitmap().delete_bitmap[{
- rowset_id, segment_id, version}] |= roaring::Roaring::read(val.data());
+ if (rowset_ids.find(rst_id) == rowset_ids.end()) {
+ continue;
+ }
+ auto seg_id = delete_bitmap_pb.segment_ids(i);
+ auto iter = tablet->tablet_meta()->delete_bitmap().delete_bitmap.find(
+ {rst_id, seg_id, version});
+ // This version of delete bitmap already exists
+ if (iter != tablet->tablet_meta()->delete_bitmap().delete_bitmap.end()) {
+ continue;
  }
+ auto bitmap = delete_bitmap_pb.segment_delete_bitmaps(i).data();
+ tablet->tablet_meta()->delete_bitmap().delete_bitmap[{rst_id, seg_id, version}] =
+ roaring::Roaring::read(bitmap);
  }
  return true;
  };

be/src/olap/storage_engine.cpp

@@ -778,8 +778,7 @@ void StorageEngine::_clean_unused_rowset_metas() {
 
 void StorageEngine::_clean_unused_delete_bitmap() {
  std::unordered_set<int64_t> removed_tablets;
- auto clean_delete_bitmap_func = [this, &removed_tablets](int64_t tablet_id, RowsetId rowset_id,
- int64_t segment_id, int64_t version,
+ auto clean_delete_bitmap_func = [this, &removed_tablets](int64_t tablet_id, int64_t version,
  const std::string& val) -> bool {
  TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id);
  if (tablet == nullptr) {

be/src/olap/tablet_meta_manager.cpp

@@ -216,66 +216,60 @@ Status TabletMetaManager::traverse_pending_publish(
  return status;
 }
 
-std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id, int64_t version,
- const RowsetId& rowset_id,
- int64_t segment_id) {
+std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id, int64_t version) {
  std::string key;
- key.reserve(56);
+ key.reserve(20);
  key.append(DELETE_BITMAP);
  put_fixed64_le(&key, BigEndian::FromHost64(tablet_id));
  put_fixed64_le(&key, BigEndian::FromHost64(version));
- put_fixed32_le(&key, BigEndian::FromHost32(rowset_id.version));
- put_fixed64_le(&key, BigEndian::FromHost64(rowset_id.hi));
- put_fixed64_le(&key, BigEndian::FromHost64(rowset_id.mi));
- put_fixed64_le(&key, BigEndian::FromHost64(rowset_id.lo));
- put_fixed64_le(&key, BigEndian::FromHost64(segment_id));
  return key;
 }
 
 void TabletMetaManager::decode_delete_bitmap_key(const string& enc_key, TTabletId* tablet_id,
- int64_t* version, RowsetId* rowset_id,
- int64_t* segment_id) {
- DCHECK_EQ(enc_key.size(), 56);
+ int64_t* version) {
+ DCHECK_EQ(enc_key.size(), 20);
  *tablet_id = BigEndian::ToHost64(UNALIGNED_LOAD64(enc_key.data() + 4));
  *version = BigEndian::ToHost64(UNALIGNED_LOAD64(enc_key.data() + 12));
- rowset_id->version = BigEndian::ToHost32(UNALIGNED_LOAD32(enc_key.data() + 20));
- rowset_id->hi = BigEndian::ToHost64(UNALIGNED_LOAD64(enc_key.data() + 24));
- rowset_id->mi = BigEndian::ToHost64(UNALIGNED_LOAD64(enc_key.data() + 32));
- rowset_id->lo = BigEndian::ToHost64(UNALIGNED_LOAD64(enc_key.data() + 40));
- *segment_id = BigEndian::ToHost64(UNALIGNED_LOAD64(enc_key.data() + 48));
 }
 
 Status TabletMetaManager::save_delete_bitmap(DataDir* store, TTabletId tablet_id,
  DeleteBitmapPtr delete_bimap, int64_t version) {
+ VLOG_NOTICE << "save delete bitmap, tablet_id:" << tablet_id << ", version: " << version;
  if (delete_bimap->delete_bitmap.empty()) {
  return Status::OK();
  }
  OlapMeta* meta = store->get_meta();
- rocksdb::WriteBatch batch;
- rocksdb::ColumnFamilyHandle* cf = meta->get_handle(META_COLUMN_FAMILY_INDEX);
+ DeleteBitmapPB delete_bitmap_pb;
  for (auto& [id, bitmap] : delete_bimap->delete_bitmap) {
  auto& rowset_id = std::get<0>(id);
  int64_t segment_id = std::get<1>(id);
- std::string key = encode_delete_bitmap_key(tablet_id, version, rowset_id, segment_id);
- std::string value(bitmap.getSizeInBytes(), '\0');
- bitmap.write(value.data());
- batch.Put(cf, key, value);
+ delete_bitmap_pb.add_rowset_ids(rowset_id.to_string());
+ delete_bitmap_pb.add_segment_ids(segment_id);
+ std::string bitmap_data(bitmap.getSizeInBytes(), '\0');
+ bitmap.write(bitmap_data.data());
+ *(delete_bitmap_pb.add_segment_delete_bitmaps()) = std::move(bitmap_data);
  }
- return meta->put(&batch);
+ std::string key = encode_delete_bitmap_key(tablet_id, version);
+ std::string val;
+ bool ok = delete_bitmap_pb.SerializeToString(&val);
+ if (!ok) {
+ auto msg = fmt::format("failed to serialize delete bitmap, tablet_id: {}, version: {}",
+ tablet_id, version);
+ LOG(WARNING) << msg;
+ return Status::InternalError(msg);
+ }
+ return meta->put(META_COLUMN_FAMILY_INDEX, key, val);
 }
 
 Status TabletMetaManager::traverse_delete_bitmap(
- OlapMeta* meta,
- std::function<bool(int64_t, RowsetId, int64_t, int64_t, const std::string&)> const& func) {
+ OlapMeta* meta, std::function<bool(int64_t, int64_t, const std::string&)> const& func) {
  auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool {
  TTabletId tablet_id;
  int64_t version;
- RowsetId rowset_id;
- int64_t segment_id;
- decode_delete_bitmap_key(key, &tablet_id, &version, &rowset_id, &segment_id);
- VLOG_NOTICE << "traverse delete bitmap, key: |" << tablet_id << "|" << rowset_id << "|"
- << segment_id << "|" << version;
- return func(tablet_id, rowset_id, segment_id, version, value);
+ decode_delete_bitmap_key(key, &tablet_id, &version);
+ VLOG_NOTICE << "traverse delete bitmap, tablet_id: " << tablet_id
+ << ", version: " << version;
+ return func(tablet_id, version, value);
  };
  Status status = meta->iterate(META_COLUMN_FAMILY_INDEX, DELETE_BITMAP, traverse_header_func);
  return status;
@@ -286,8 +280,8 @@ Status TabletMetaManager::remove_old_version_delete_bitmap(DataDir* store, TTabl
  OlapMeta* meta = store->get_meta();
  rocksdb::WriteBatch batch;
  rocksdb::ColumnFamilyHandle* cf = meta->get_handle(META_COLUMN_FAMILY_INDEX);
- auto lower_key = encode_delete_bitmap_key(tablet_id, 0, RowsetId(), 0);
- auto upper_key = encode_delete_bitmap_key(tablet_id, version + 1, RowsetId(), 0);
+ auto lower_key = encode_delete_bitmap_key(tablet_id, 0);
+ auto upper_key = encode_delete_bitmap_key(tablet_id, version + 1);
  batch.DeleteRange(cf, lower_key, upper_key);
  LOG(INFO) << "remove old version delete bitmap, tablet_id: " << tablet_id
  << " version: " << version;
@@ -298,8 +292,8 @@ Status TabletMetaManager::remove_delete_bitmap_by_tablet_id(DataDir* store, TTab
  OlapMeta* meta = store->get_meta();
  rocksdb::WriteBatch batch;
  rocksdb::ColumnFamilyHandle* cf = meta->get_handle(META_COLUMN_FAMILY_INDEX);
- auto lower_key = encode_delete_bitmap_key(tablet_id, 0, RowsetId(), 0);
- auto upper_key = encode_delete_bitmap_key(tablet_id, INT64_MAX, RowsetId(), 0);
+ auto lower_key = encode_delete_bitmap_key(tablet_id, 0);
+ auto upper_key = encode_delete_bitmap_key(tablet_id, INT64_MAX);
  batch.DeleteRange(cf, lower_key, upper_key);
  LOG(INFO) << "remove delete bitmap by tablet_id, tablet_id: " << tablet_id;
  return meta->put(&batch);

be/src/olap/tablet_meta_manager.h

@@ -75,16 +75,13 @@ class TabletMetaManager {
  static Status save_delete_bitmap(DataDir* store, TTabletId tablet_id,
  DeleteBitmapPtr delete_bimap, int64_t version);
 
- static Status traverse_delete_bitmap(OlapMeta* meta,
- std::function<bool(int64_t, RowsetId, int64_t, int64_t,
- const std::string&)> const& func);
+ static Status traverse_delete_bitmap(
+ OlapMeta* meta, std::function<bool(int64_t, int64_t, const std::string&)> const& func);
 
- static std::string encode_delete_bitmap_key(TTabletId tablet_id, int64_t version,
- const RowsetId& rowset_id, int64_t segment_id);
+ static std::string encode_delete_bitmap_key(TTabletId tablet_id, int64_t version);
 
  static void decode_delete_bitmap_key(const string& enc_key, TTabletId* tablet_id,
- int64_t* version, RowsetId* rowset_id,
- int64_t* segment_id);
+ int64_t* version);
  static Status remove_old_version_delete_bitmap(DataDir* store, TTabletId tablet_id,
  int64_t version);
 

be/test/olap/tablet_meta_manager_test.cpp

@@ -20,7 +20,9 @@
 #include <gen_cpp/olap_file.pb.h>
 #include <gtest/gtest-message.h>
 #include <gtest/gtest-test-part.h>
+#include <gtest/gtest.h>
 #include <json2pb/json_to_pb.h>
+#include <stddef.h>
 
 #include <filesystem>
 #include <fstream>
@@ -112,22 +114,13 @@ TEST_F(TabletMetaManagerTest, TestLoad) {
 TEST_F(TabletMetaManagerTest, TestDeleteBimapEncode) {
  TTabletId tablet_id = 1234;
  int64_t version = 456;
- RowsetId rowset_id;
- rowset_id.init(2, 777, 888, 999);
- int64_t segment_id = 5;
- std::string key =
- TabletMetaManager::encode_delete_bitmap_key(tablet_id, version, rowset_id, segment_id);
+ std::string key = TabletMetaManager::encode_delete_bitmap_key(tablet_id, version);
 
  TTabletId de_tablet_id;
  int64_t de_version;
- RowsetId de_rowset_id;
- int64_t de_segment_id;
- TabletMetaManager::decode_delete_bitmap_key(key, &de_tablet_id, &de_version, &de_rowset_id,
- &de_segment_id);
+ TabletMetaManager::decode_delete_bitmap_key(key, &de_tablet_id, &de_version);
  EXPECT_EQ(tablet_id, de_tablet_id);
  EXPECT_EQ(version, de_version);
- EXPECT_EQ(rowset_id, de_rowset_id);
- EXPECT_EQ(segment_id, de_segment_id);
 }
 
 TEST_F(TabletMetaManagerTest, TestSaveDeleteBimap) {
@@ -150,28 +143,35 @@ TEST_F(TabletMetaManagerTest, TestSaveDeleteBimap) {
  TabletMetaManager::save_delete_bitmap(_data_dir, test_tablet_id, dbmp, ver);
  }
  size_t num_keys = 0;
- auto load_delete_bitmap_func = [&](int64_t tablet_id, RowsetId rowset_id, int64_t segment_id,
- int64_t version, const string& val) {
+ auto load_delete_bitmap_func = [&](int64_t tablet_id, int64_t version, const string& val) {
  EXPECT_EQ(tablet_id, test_tablet_id);
- auto iter = dbmp->delete_bitmap.find({rowset_id, segment_id, 0});
- EXPECT_NE(iter, dbmp->delete_bitmap.end());
- auto bitmap = roaring::Roaring::read(val.data());
- EXPECT_EQ(bitmap.cardinality(), 10);
+ DeleteBitmapPB delete_bitmap_pb;
+ delete_bitmap_pb.ParseFromString(val);
+ int rst_ids_size = delete_bitmap_pb.rowset_ids_size();
+ int seg_ids_size = delete_bitmap_pb.segment_ids_size();
+ int seg_maps_size = delete_bitmap_pb.segment_delete_bitmaps_size();
+ EXPECT_EQ(rst_ids_size, max_rst_id * max_seg_id);
+ EXPECT_EQ(seg_ids_size, rst_ids_size);
+ EXPECT_EQ(seg_maps_size, rst_ids_size);
+ for (size_t i = 0; i < rst_ids_size; i++) {
+ auto bitmap = roaring::Roaring::read(delete_bitmap_pb.segment_delete_bitmaps(i).data());
+ EXPECT_EQ(bitmap.cardinality(), 10);
+ }
  ++num_keys;
  return true;
  };
  TabletMetaManager::traverse_delete_bitmap(_data_dir->get_meta(), load_delete_bitmap_func);
- EXPECT_EQ(num_keys, max_rst_id * max_seg_id * max_version);
+ EXPECT_EQ(num_keys, max_version);
 
  num_keys = 0;
  TabletMetaManager::remove_old_version_delete_bitmap(_data_dir, test_tablet_id, 100);
  TabletMetaManager::traverse_delete_bitmap(_data_dir->get_meta(), load_delete_bitmap_func);
- EXPECT_EQ(num_keys, max_rst_id * max_seg_id * (max_version - 101));
+ EXPECT_EQ(num_keys, max_version - 101);
 
  num_keys = 0;
  TabletMetaManager::remove_old_version_delete_bitmap(_data_dir, test_tablet_id, 200);
  TabletMetaManager::traverse_delete_bitmap(_data_dir->get_meta(), load_delete_bitmap_func);
- EXPECT_EQ(num_keys, max_rst_id * max_seg_id * (max_version - 201));
+ EXPECT_EQ(num_keys, max_version - 201);
 
  num_keys = 0;
  TabletMetaManager::remove_delete_bitmap_by_tablet_id(_data_dir, test_tablet_id);