Skip to content

[fix](cloud-mow) get delete bitmap update lock should create a new Transaction when encounter TXN_TOO_OLD #44691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
hust-hhb wants to merge 2 commits into from
Closed

[fix](cloud-mow) get delete bitmap update lock should create a new Transaction when encounter TXN_TOO_OLD #44691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f80108
[fix](cloud-mow) get delete bitmap update lock should create a new Tr…
hust-hhb Dec 4, 2024
e6b63ec
edit
hust-hhb Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions cloud/src/meta-service/meta_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2124,10 +2124,30 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
// this request is from fe when it commits txn for MOW table, we send the compaction stats
// along with the GetDeleteBitmapUpdateLockResponse which will be sent to BE later to let
// BE eliminate unnecessary sync_rowsets() calls if possible
int64_t count = 0;
for (const auto& tablet_index : request->tablet_indexes()) {
TabletIndexPB idx(tablet_index);
TabletStatsPB tablet_stat;
int64_t retry = 0;
internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, false);
count++;
TEST_SYNC_POINT_CALLBACK("get_tablet_stats_code", &count, &code);
while (retry < 3 && code == MetaServiceCode::KV_TXN_TOO_OLD) {
code = MetaServiceCode::OK;
txn = nullptr;
retry++;
err = txn_kv_->create_txn(&txn);
Copy link
Contributor

@gavinchou gavinchou Dec 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

overwrite the txn cannot modify the "delete_bitmap_upldate_lock"

if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::CREATE>(err);
ss << "failed to init txn, retry=" << retry;
msg = ss.str();
return;
}
internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat,
false);
LOG(INFO) << "retry get tablet stats, tablet=" << idx.tablet_id()
<< ", retry=" << retry;
}
if (code != MetaServiceCode::OK) {
response->clear_base_compaction_cnts();
response->clear_cumulative_compaction_cnts();
Expand All @@ -2141,6 +2161,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt());
response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt());
response->add_cumulative_points(tablet_stat.cumulative_point());
LOG(INFO) << "finish get tablet stats for tablet=" << idx.tablet_id()
<< ",count=" << count;
}
}

Expand Down
90 changes: 87 additions & 3 deletions cloud/test/meta_service_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4474,7 +4474,7 @@ TEST(MetaServiceTest, GetTabletStatsTest) {
EXPECT_EQ(res.tablet_stats(0).num_segments(), 4);
}

TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
TEST(MetaServiceTest, GetDeleteBitmapUpdateLock1) {
auto meta_service = get_meta_service();

brpc::Controller cntl;
Expand Down Expand Up @@ -4516,6 +4516,90 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
}

TEST(MetaServiceTest, GetDeleteBitmapUpdateLock2) {
auto meta_service = get_meta_service();

SyncPoint::get_instance()->enable_processing();
size_t index = 0;
SyncPoint::get_instance()->set_call_back("get_tablet_stats_code", [&](auto&& args) {
auto* count = try_any_cast<int64_t*>(args[0]);
LOG(INFO) << "GET_DELETE_BITMAP_UPDATE_LOCK_CODE,index=" << index << ",count=" << *count;
if (*count > 1 && ++index < 4) {
*doris::try_any_cast<MetaServiceCode*>(args[1]) = MetaServiceCode::KV_TXN_TOO_OLD;
}
});

extern std::string get_instance_id(const std::shared_ptr<ResourceManager>& rc_mgr,
const std::string& cloud_unique_id);
auto instance_id = get_instance_id(meta_service->resource_mgr(), "test_cloud_unique_id");

int64_t table_id = 1;
int64_t index_id_1 = 1;
int64_t partition_id_1 = 1;
int64_t tablet_id_1 = 100;

int64_t index_id_2 = 2;
int64_t partition_id_2 = 2;
int64_t tablet_id_2 = 101;

//store tablet stats
std::unique_ptr<Transaction> txn;
ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);

std::string stats_key_1 =
stats_tablet_key({instance_id, table_id, index_id_1, partition_id_1, tablet_id_1});
TabletStatsPB stats_1;
stats_1.set_base_compaction_cnt(9);
stats_1.set_cumulative_compaction_cnt(19);
stats_1.set_cumulative_point(20);
txn->put(stats_key_1, stats_1.SerializeAsString());

std::string stats_key_2 =
stats_tablet_key({instance_id, table_id, index_id_2, partition_id_2, tablet_id_2});
TabletStatsPB stats_2;
stats_2.set_base_compaction_cnt(9);
stats_2.set_cumulative_compaction_cnt(19);
stats_2.set_cumulative_point(20);
txn->put(stats_key_2, stats_2.SerializeAsString());
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);

brpc::Controller cntl;
GetDeleteBitmapUpdateLockRequest req;
GetDeleteBitmapUpdateLockResponse res;
req.set_cloud_unique_id("test_cloud_unique_id");
req.set_table_id(1);
req.add_partition_ids(1);
req.add_partition_ids(2);
req.set_expiration(5);
req.set_lock_id(888);
req.set_initiator(-1);

req.set_require_compaction_stats(true);

auto tablet_index_1 = req.add_tablet_indexes();
tablet_index_1->set_table_id(table_id);
tablet_index_1->set_index_id(index_id_1);
tablet_index_1->set_partition_id(partition_id_1);
tablet_index_1->set_tablet_id(tablet_id_1);

auto tablet_index_2 = req.add_tablet_indexes();
tablet_index_2->set_table_id(table_id);
tablet_index_2->set_index_id(index_id_2);
tablet_index_2->set_partition_id(partition_id_2);
tablet_index_2->set_tablet_id(tablet_id_2);

meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
ASSERT_EQ(res.base_compaction_cnts().size(), 2);
for (const auto& base_compaction_cnt : res.base_compaction_cnts()) {
ASSERT_EQ(base_compaction_cnt, 9);
}

SyncPoint::get_instance()->disable_processing();
SyncPoint::get_instance()->clear_all_call_backs();
}

static std::string generate_random_string(int length) {
std::string char_set = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
std::random_device rd;
Expand Down Expand Up @@ -5223,7 +5307,7 @@ TEST(MetaServiceTest, GetDeleteBitmapWithRetryTest2) {
SyncPoint::get_instance()->set_call_back("get_delete_bitmap_err", [&](auto&& args) {
auto* round = try_any_cast<int64_t*>(args[0]);
LOG(INFO) << "GET_DELETE_BITMAP_CODE,index=" << index << ",round=" << *round;
if (*round > 2 && ++index < 2) {
if (*round > 2 && ++index < 4) {
*try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_TOO_OLD;
}
});
Expand Down Expand Up @@ -5303,7 +5387,7 @@ TEST(MetaServiceTest, GetDeleteBitmapWithRetryTest3) {
SyncPoint::get_instance()->set_call_back("get_delete_bitmap_err", [&](auto&& args) {
auto* round = try_any_cast<int64_t*>(args[0]);
LOG(INFO) << "GET_DELETE_BITMAP_CODE,index=" << index << ",round=" << *round;
if (*round > 2 && ++index < 2) {
if (*round > 2 && ++index < 4) {
*try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_TOO_OLD;
}
});
Expand Down
Loading