Skip to content

Commit

Permalink
cluster: use do_run_test() in cloudcheck
Browse files Browse the repository at this point in the history 
The `self-test status` will now have the properly formatted
start and end times, regardless of whether a test was successful
or not.
  • Loading branch information
@WillemKauf
WillemKauf committed Jul 12, 2024
1 parent 3c4468a commit e7a3cb8
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 64 deletions.
88 changes: 34 additions & 54 deletions src/v/cluster/self_test/cloudcheck.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,9 @@ ss::future<std::vector<self_test_result>> cloudcheck::run_benchmarks() {
: std::optional<iobuf>{};

// Test Put
auto upload_test_result = co_await verify_upload(
bucket, self_test_key, payload);
auto verify_test_result = co_await do_run_test(
&cloudcheck::verify_upload, bucket, self_test_key, payload);
auto upload_test_result = verify_test_result.test_result;
const bool is_uploaded
= (!upload_test_result.warning && !upload_test_result.error);

Expand All @@ -126,7 +127,8 @@ ss::future<std::vector<self_test_result>> cloudcheck::run_benchmarks() {
const std::optional<cloud_storage_clients::object_key> list_prefix
= (is_uploaded) ? self_test_prefix
: std::optional<cloud_storage_clients::object_key>{};
auto list_test_result_pair = co_await verify_list(bucket, list_prefix);
auto list_test_result_pair = co_await do_run_test(
&cloudcheck::verify_list, bucket, list_prefix, num_default_objects);
auto& [object_list, list_test_result] = list_test_result_pair;
if (is_uploaded && object_list) {
// Check that uploaded object exists in object_list contents.
Expand Down Expand Up @@ -173,8 +175,8 @@ ss::future<std::vector<self_test_result>> cloudcheck::run_benchmarks() {

const std::optional<cloud_storage_clients::object_key> download_key
= (is_uploaded) ? self_test_key : get_min_object_key();
auto download_test_result_pair = co_await verify_download(
bucket, download_key);
auto download_test_result_pair = co_await do_run_test(
&cloudcheck::verify_download, bucket, download_key);
auto& [downloaded_object, download_test_result] = download_test_result_pair;
if (is_uploaded && downloaded_object) {
auto& downloaded_buf = downloaded_object.value();
Expand All @@ -187,16 +189,19 @@ ss::future<std::vector<self_test_result>> cloudcheck::run_benchmarks() {
results.push_back(std::move(download_test_result));

// Test Head
auto head_test_result = co_await verify_head(bucket, download_key);
results.push_back(std::move(head_test_result));
auto head_test_result = co_await do_run_test(
&cloudcheck::verify_head, bucket, download_key);
results.push_back(std::move(head_test_result.test_result));

// Test Delete
auto delete_test_result = co_await verify_delete(bucket, self_test_key);
results.push_back(std::move(delete_test_result));
auto delete_test_result = co_await do_run_test(
&cloudcheck::verify_delete, bucket, self_test_key);
results.push_back(std::move(delete_test_result.test_result));

// Test Deletes
auto deletes_test_result = co_await verify_deletes(bucket);
results.push_back(std::move(deletes_test_result));
auto deletes_test_result = co_await do_run_test(
&cloudcheck::verify_deletes, bucket, num_default_objects);
results.push_back(std::move(deletes_test_result.test_result));

co_return results;
}
Expand Down Expand Up @@ -236,7 +241,7 @@ cloud_storage::download_request cloudcheck::make_download_request(
return download_request;
}

ss::future<self_test_result> cloudcheck::verify_upload(
ss::future<cloudcheck::verify_upload_result> cloudcheck::verify_upload(
cloud_storage_clients::bucket_name bucket,
cloud_storage_clients::object_key key,
const std::optional<iobuf>& payload) {
Expand All @@ -253,7 +258,6 @@ ss::future<self_test_result> cloudcheck::verify_upload(
co_return result;
}

const auto start = ss::lowres_clock::now();
try {
auto rtc = retry_chain_node(_opts.timeout, _opts.backoff, &_rtc);
cloud_storage::upload_request upload_request = make_upload_request(
Expand All @@ -277,14 +281,10 @@ ss::future<self_test_result> cloudcheck::verify_upload(
result.error = e.what();
}

const auto end = ss::lowres_clock::now();
result.duration = end - start;

co_return result;
}

ss::future<std::pair<cloud_storage::remote::list_result, self_test_result>>
cloudcheck::verify_list(
ss::future<cloudcheck::verify_list_result> cloudcheck::verify_list(
cloud_storage_clients::bucket_name bucket,
std::optional<cloud_storage_clients::object_key> prefix,
size_t max_keys) {
Expand All @@ -293,17 +293,16 @@ cloudcheck::verify_list(

if (_cancelled) {
result.warning = "Run was manually cancelled.";
co_return std::make_pair(
cloud_storage_clients::error_outcome::fail, result);
co_return verify_list_result{
cloud_storage_clients::error_outcome::fail, result};
}

if (!_remote_read_enabled) {
result.error = "Remote read is not enabled for this cluster.";
co_return std::make_pair(
cloud_storage_clients::error_outcome::fail, result);
co_return verify_list_result{
cloud_storage_clients::error_outcome::fail, result};
}

const auto start = ss::lowres_clock::now();
try {
auto rtc = retry_chain_node(_opts.timeout, _opts.backoff, &_rtc);
const cloud_storage::remote::list_result object_list
Expand All @@ -314,19 +313,16 @@ cloudcheck::verify_list(
result.error = "Failed to list objects in cloud storage.";
}

co_return std::make_pair(std::move(object_list), std::move(result));
co_return verify_list_result{std::move(object_list), std::move(result)};
} catch (const std::exception& e) {
result.error = e.what();
}

const auto end = ss::lowres_clock::now();
result.duration = end - start;

co_return std::make_pair(
cloud_storage_clients::error_outcome::fail, std::move(result));
co_return verify_list_result{
cloud_storage_clients::error_outcome::fail, std::move(result)};
}

ss::future<self_test_result> cloudcheck::verify_head(
ss::future<cloudcheck::verify_head_result> cloudcheck::verify_head(
cloud_storage_clients::bucket_name bucket,
std::optional<cloud_storage_clients::object_key> key) {
auto result = self_test_result{
Expand All @@ -348,7 +344,6 @@ ss::future<self_test_result> cloudcheck::verify_head(
co_return result;
}

const auto start = ss::lowres_clock::now();
try {
auto rtc = retry_chain_node(_opts.timeout, _opts.backoff, &_rtc);
const cloud_storage::download_result head_result
Expand All @@ -374,38 +369,33 @@ ss::future<self_test_result> cloudcheck::verify_head(
result.error = e.what();
}

const auto end = ss::lowres_clock::now();
result.duration = end - start;

co_return result;
}

ss::future<std::pair<std::optional<iobuf>, self_test_result>>
cloudcheck::verify_download(
ss::future<cloudcheck::verify_download_result> cloudcheck::verify_download(
cloud_storage_clients::bucket_name bucket,
std::optional<cloud_storage_clients::object_key> key) {
auto result = self_test_result{
.name = _opts.name, .info = "Get", .test_type = "cloud"};

if (_cancelled) {
result.warning = "Run was manually cancelled.";
co_return std::make_pair(std::nullopt, result);
co_return verify_download_result{std::nullopt, result};
}

if (!_remote_read_enabled) {
result.error = "Remote read is not enabled for this cluster.";
co_return std::make_pair(std::nullopt, result);
co_return verify_download_result{std::nullopt, result};
}

if (!key) {
result.warning = "Could not download from cloud storage (no file was "
"found in the bucket).";
co_return std::make_pair(std::nullopt, result);
co_return verify_download_result{std::nullopt, result};
}

std::optional<iobuf> result_payload = std::nullopt;

const auto start = ss::lowres_clock::now();
try {
iobuf download_payload;
auto rtc = retry_chain_node(_opts.timeout, _opts.backoff, &_rtc);
Expand Down Expand Up @@ -433,13 +423,11 @@ cloudcheck::verify_download(
result.error = e.what();
}

const auto end = ss::lowres_clock::now();
result.duration = end - start;

co_return std::make_pair(std::move(result_payload), std::move(result));
co_return verify_download_result{
std::move(result_payload), std::move(result)};
}

ss::future<self_test_result> cloudcheck::verify_delete(
ss::future<cloudcheck::verify_delete_result> cloudcheck::verify_delete(
cloud_storage_clients::bucket_name bucket,
cloud_storage_clients::object_key key) {
auto result = self_test_result{
Expand All @@ -455,7 +443,6 @@ ss::future<self_test_result> cloudcheck::verify_delete(
co_return result;
}

const auto start = ss::lowres_clock::now();
try {
auto rtc = retry_chain_node(_opts.timeout, _opts.backoff, &_rtc);
const cloud_storage::upload_result delete_result
Expand All @@ -476,13 +463,10 @@ ss::future<self_test_result> cloudcheck::verify_delete(
result.error = e.what();
}

const auto end = ss::lowres_clock::now();
result.duration = end - start;

co_return result;
}

ss::future<self_test_result> cloudcheck::verify_deletes(
ss::future<cloudcheck::verify_deletes_result> cloudcheck::verify_deletes(
cloud_storage_clients::bucket_name bucket, size_t num_objects) {
auto result = self_test_result{
.name = _opts.name, .info = "Plural Delete", .test_type = "cloud"};
Expand All @@ -506,7 +490,6 @@ ss::future<self_test_result> cloudcheck::verify_deletes(
co_await verify_upload(bucket, key, make_random_payload());
}

const auto start = ss::lowres_clock::now();
try {
auto rtc = retry_chain_node(_opts.timeout, _opts.backoff, &_rtc);
const cloud_storage::upload_result delete_result
Expand All @@ -528,9 +511,6 @@ ss::future<self_test_result> cloudcheck::verify_deletes(
result.error = e.what();
}

const auto end = ss::lowres_clock::now();
result.duration = end - start;

co_return result;
}

Expand Down
46 changes: 36 additions & 10 deletions src/v/cluster/self_test/cloudcheck.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,42 +82,68 @@ class cloudcheck {
typename R = std::invoke_result_t<Test, cloudcheck, Args...>>
R do_run_test(Test test, Args&&... args);

struct verify_upload_result {
self_test_result test_result;
};

// Verify that uploading (Put: write operation) to cloud storage works.
ss::future<self_test_result> verify_upload(
ss::future<verify_upload_result> verify_upload(
cloud_storage_clients::bucket_name bucket,
cloud_storage_clients::object_key key,
const std::optional<iobuf>& payload);

struct verify_list_result {
cloud_storage::remote::list_result list_result;
self_test_result test_result;
};

// Verify that listing (List: read operation) from cloud storage works.
ss::future<std::pair<cloud_storage::remote::list_result, self_test_result>>
verify_list(
ss::future<verify_list_result> verify_list(
cloud_storage_clients::bucket_name bucket,
std::optional<cloud_storage_clients::object_key> prefix,
size_t max_keys = 5);
size_t max_keys = num_default_objects);

struct verify_head_result {
self_test_result test_result;
};

// Verify that checking if an object exists (Head: read operation) from
// cloud storage works.
ss::future<self_test_result> verify_head(
ss::future<verify_head_result> verify_head(
cloud_storage_clients::bucket_name bucket,
std::optional<cloud_storage_clients::object_key> key);

struct verify_download_result {
std::optional<iobuf> buf;
self_test_result test_result;
};

// Verify that downloading (Get: read operation) from cloud storage works.
ss::future<std::pair<std::optional<iobuf>, self_test_result>>
verify_download(
ss::future<verify_download_result> verify_download(
cloud_storage_clients::bucket_name bucket,
std::optional<cloud_storage_clients::object_key> key);

struct verify_delete_result {
self_test_result test_result;
};

// Verify that deleting (Delete: write operation) from cloud storage works.
ss::future<self_test_result> verify_delete(
ss::future<verify_delete_result> verify_delete(
cloud_storage_clients::bucket_name bucket,
cloud_storage_clients::object_key key);

struct verify_deletes_result {
self_test_result test_result;
};

// Verify that deleting multiple (Plural Delete: write operation) from cloud
// storage works.
ss::future<self_test_result> verify_deletes(
cloud_storage_clients::bucket_name bucket, size_t num_objects = 5);
ss::future<verify_deletes_result> verify_deletes(
cloud_storage_clients::bucket_name bucket,
size_t num_objects = num_default_objects);

private:
static constexpr size_t num_default_objects = 5;
bool _remote_read_enabled{false};
bool _remote_write_enabled{false};
bool _cancelled{false};
Expand Down

0 comments on commit e7a3cb8

Please sign in to comment.