remove delete range

liaoxin01 · Jul 6, 2023 · f5319fc · f5319fc
1 parent d24cb60
commit f5319fc
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 25 deletions.
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
@@ -793,7 +793,7 @@ void StorageEngine::_clean_unused_delete_bitmap() {
  for (auto data_dir : data_dirs) {
  TabletMetaManager::traverse_delete_bitmap(data_dir->get_meta(), clean_delete_bitmap_func);
  for (auto id : removed_tablets) {
- TabletMetaManager::remove_delete_bitmap_by_tablet_id(data_dir, id);
+ TabletMetaManager::remove_old_version_delete_bitmap(data_dir, id, INT64_MAX);
  }
  LOG(INFO) << "removed invalid delete bitmap from dir: " << data_dir->path()
  << ", deleted tablets size: " << removed_tablets.size();

diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp
@@ -225,6 +225,14 @@ std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id, int
  return key;
 }
 
+std::string TabletMetaManager::encode_delete_bitmap_key(TTabletId tablet_id) {
+ std::string key;
+ key.reserve(12);
+ key.append(DELETE_BITMAP);
+ put_fixed64_le(&key, BigEndian::FromHost64(tablet_id));
+ return key;
+}
+
 void TabletMetaManager::decode_delete_bitmap_key(const string& enc_key, TTabletId* tablet_id,
  int64_t* version) {
  DCHECK_EQ(enc_key.size(), 20);
@@ -271,32 +279,29 @@ Status TabletMetaManager::traverse_delete_bitmap(
  << ", version: " << version;
  return func(tablet_id, version, value);
  };
- Status status = meta->iterate(META_COLUMN_FAMILY_INDEX, DELETE_BITMAP, traverse_header_func);
- return status;
+ return meta->iterate(META_COLUMN_FAMILY_INDEX, DELETE_BITMAP, traverse_header_func);
 }
 
 Status TabletMetaManager::remove_old_version_delete_bitmap(DataDir* store, TTabletId tablet_id,
  int64_t version) {
  OlapMeta* meta = store->get_meta();
- rocksdb::WriteBatch batch;
- rocksdb::ColumnFamilyHandle* cf = meta->get_handle(META_COLUMN_FAMILY_INDEX);
- auto lower_key = encode_delete_bitmap_key(tablet_id, 0);
- auto upper_key = encode_delete_bitmap_key(tablet_id, version + 1);
- batch.DeleteRange(cf, lower_key, upper_key);
- LOG(INFO) << "remove old version delete bitmap, tablet_id: " << tablet_id
- << " version: " << version;
- return meta->put(&batch);
-}
+ std::string begin_key = encode_delete_bitmap_key(tablet_id);
+ std::string end_key = encode_delete_bitmap_key(tablet_id, version);
 
-Status TabletMetaManager::remove_delete_bitmap_by_tablet_id(DataDir* store, TTabletId tablet_id) {
- OlapMeta* meta = store->get_meta();
- rocksdb::WriteBatch batch;
- rocksdb::ColumnFamilyHandle* cf = meta->get_handle(META_COLUMN_FAMILY_INDEX);
- auto lower_key = encode_delete_bitmap_key(tablet_id, 0);
- auto upper_key = encode_delete_bitmap_key(tablet_id, INT64_MAX);
- batch.DeleteRange(cf, lower_key, upper_key);
- LOG(INFO) << "remove delete bitmap by tablet_id, tablet_id: " << tablet_id;
- return meta->put(&batch);
+ std::vector<std::string> remove_keys;
+ auto get_remove_keys_func = [&](const std::string& key, const std::string& val) -> bool {
+ // include end_key
+ if (key > end_key) {
+ return false;
+ }
+ remove_keys.push_back(key);
+ return true;
+ };
+ LOG(INFO) << "remove old version delete bitmap, tablet_id: " << tablet_id
+ << " version: " << version << " removed keys size: " << remove_keys.size();
+ ;
+ RETURN_IF_ERROR(meta->iterate(META_COLUMN_FAMILY_INDEX, begin_key, get_remove_keys_func));
+ return meta->remove(META_COLUMN_FAMILY_INDEX, remove_keys);
 }
 
 } // namespace doris
diff --git a/be/src/olap/tablet_meta_manager.h b/be/src/olap/tablet_meta_manager.h
@@ -79,13 +79,12 @@ class TabletMetaManager {
  OlapMeta* meta, std::function<bool(int64_t, int64_t, const std::string&)> const& func);
 
  static std::string encode_delete_bitmap_key(TTabletId tablet_id, int64_t version);
+ static std::string encode_delete_bitmap_key(TTabletId tablet_id);
 
  static void decode_delete_bitmap_key(const string& enc_key, TTabletId* tablet_id,
  int64_t* version);
  static Status remove_old_version_delete_bitmap(DataDir* store, TTabletId tablet_id,
  int64_t version);
-
- static Status remove_delete_bitmap_by_tablet_id(DataDir* store, TTabletId tablet_id);
 };
 
 } // namespace doris
diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp
@@ -384,7 +384,7 @@ void BackendService::check_storage_format(TCheckStorageFormatResult& result) {
 
 void BackendService::ingest_binlog(TIngestBinlogResult& result,
  const TIngestBinlogRequest& request) {
- constexpr uint64_t kMaxTimeoutMs = 10000;
+ constexpr uint64_t kMaxTimeoutMs = 1000;
  TStatus tstatus;
  Defer defer {[&result, &tstatus]() { result.__set_status(tstatus); }};
 

diff --git a/be/test/olap/tablet_meta_manager_test.cpp b/be/test/olap/tablet_meta_manager_test.cpp
@@ -174,7 +174,7 @@ TEST_F(TabletMetaManagerTest, TestSaveDeleteBimap) {
  EXPECT_EQ(num_keys, max_version - 201);
 
  num_keys = 0;
- TabletMetaManager::remove_delete_bitmap_by_tablet_id(_data_dir, test_tablet_id);
+ TabletMetaManager::remove_old_version_delete_bitmap(_data_dir, test_tablet_id, INT64_MAX);
  TabletMetaManager::traverse_delete_bitmap(_data_dir->get_meta(), load_delete_bitmap_func);
  EXPECT_EQ(num_keys, 0);
 }