From 3fb48c6a215b920c44676d2c73f5bef62e30d054 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 21:49:42 +0100 Subject: [PATCH 1/9] add replication structures --- src/CMakeLists.txt | 3 ++- src/reduct/bucket.h | 11 +++++++--- src/reduct/client.h | 44 ++++++++++++++++++++++++++++++---------- src/reduct/diagnostics.h | 33 ++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 src/reduct/diagnostics.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 206f092..10a0925 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,8 @@ set(PUBLIC_HEADERS reduct/client.h reduct/error.h reduct/http_options.h - reduct/result.h) + reduct/result.h + reduct/diagnostics.h) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") diff --git a/src/reduct/bucket.h b/src/reduct/bucket.h index 9859e36..73eb52a 100644 --- a/src/reduct/bucket.h +++ b/src/reduct/bucket.h @@ -1,4 +1,4 @@ -// Copyright 2022-2023 Alexey Timin +// Copyright 2022-2024 Alexey Timin #ifndef REDUCT_CPP_BUCKET_H #define REDUCT_CPP_BUCKET_H @@ -16,9 +16,14 @@ #include "reduct/result.h" namespace reduct { - /** - * Provides Bucket HTTP API + * @class IBucket + * @brief Represents a bucket for storing and retrieving data. + * + * A Bucket object allows you to interact with a specific bucket in the storage system. + * You can perform operations like getting the bucket settings, updating the settings, + * getting bucket information, retrieving the list of entries, writing records to the bucket, + * and reading records from the bucket. */ class IBucket { public: diff --git a/src/reduct/client.h b/src/reduct/client.h index b1df8b5..9e34270 100644 --- a/src/reduct/client.h +++ b/src/reduct/client.h @@ -1,4 +1,4 @@ -// Copyright 2022 Alexey Timin +// Copyright 2022-2024 Alexey Timin #ifndef REDUCT_CPP_CLIENT_H #define REDUCT_CPP_CLIENT_H @@ -10,6 +10,7 @@ #include #include +#include "diagnostics.h" #include "reduct/bucket.h" #include "reduct/error.h" #include "reduct/http_options.h" @@ -151,19 +152,40 @@ class IClient { * Makes a GET request to the '/me' endpoint using the client object stored as a member variable. * * @return A Result object containing a FullTokenInfo object or an Error object. - * - * Example: - * - * MyClass obj; - * auto [token_info, err] = obj.Me(); - * if (err) { - * std::cerr << err << std::endl; - * } else { - * std::cout << token_info.name << std::endl; - * } */ [[nodiscard]] virtual Result Me() const noexcept = 0; + struct ReplicationInfo { + std::string name; // Replication name + bool is_active; // Remote instance is available and replication is active + bool is_provisioned; // Replication settings + uint64_t pending_records; // Number of records pending replication + + bool operator<=>(const ReplicationInfo&) const = default; + }; + + struct ReplicationSettings { + std::string src_bucket; // Source bucket + std::string dst_bucket; // Destination bucket + std::string dst_host; // Destination host URL (e.g. https://reductstore.com) + std::string dst_token; // Destination access token + std::vector + entries; // Entries to replicate. If empty, all entries are replicated. Wildcards are supported. + IBucket::LabelMap include; // Labels to include + IBucket::LabelMap exclude; // Labels to exclude + + bool operator<=>(const ReplicationSettings&) const = default; + }; + + struct ReplicationFullInfo { + ReplicationInfo info; // Replication info + ReplicationSettings settings; // Replication settings + Diagnostics diagnostics; // Diagnostics + + bool operator ==(const ReplicationFullInfo&) const = default; + }; + + /** * @brief Build a client * @param url URL of React Storage diff --git a/src/reduct/diagnostics.h b/src/reduct/diagnostics.h new file mode 100644 index 0000000..4d36cbb --- /dev/null +++ b/src/reduct/diagnostics.h @@ -0,0 +1,33 @@ +// Copyright 2024 Alexey Timin + +#ifndef REDUCT_CPP_DIAGNOSTICS_H +#define REDUCT_CPP_DIAGNOSTICS_H + +#include +#include + +namespace reduct { + +struct DiagnosticsError { + uint64_t count; // Count of errors + std::string last_message; // Last error message + + bool operator<=>(const DiagnosticsError&) const = default; +}; + +struct DiagnosticsItem { + uint64_t ok; // Count of successful operations + uint64_t errored; // Count of errored operations + std::map errors; // Map of error codes to DiagnosticsError + + bool operator==(const DiagnosticsItem&) const = default; +}; + +struct Diagnostics { + DiagnosticsItem hourly; // Hourly diagnostics item + + bool operator==(const Diagnostics&) const = default; +}; + +} // namespace reduct +#endif // REDUCT_CPP_DIAGNOSTICS_H From ad3cc63d90e84c4ed52cab21599c40d994040639 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:00:24 +0100 Subject: [PATCH 2/9] implement replicaion endpoint --- src/reduct/client.cc | 42 ++++++++++++++++ src/reduct/client.h | 59 +++++++++++++++++++--- src/reduct/internal/serialisation.cc | 75 ++++++++++++++++++++++++++-- src/reduct/internal/serialisation.h | 29 +++++++++-- 4 files changed, 192 insertions(+), 13 deletions(-) diff --git a/src/reduct/client.cc b/src/reduct/client.cc index ad0dd24..0d04165 100644 --- a/src/reduct/client.cc +++ b/src/reduct/client.cc @@ -187,6 +187,48 @@ class Client : public IClient { } } + Result> GetReplicationList() const noexcept override { + auto [body, err] = client_->Get("/replication"); + if (err) { + return {{}, std::move(err)}; + } + + try { + nlohmann::json data = nlohmann::json::parse(body); + return internal::ParseReplicationList(data); + } catch (const std::exception& e) { + return {{}, Error{.code = -1, .message = e.what()}}; + } + } + + Result GetReplication(std::string_view name) const noexcept override { + auto [body, err] = client_->Get(fmt::format("/replication/{}", name)); + if (err) { + return {{}, std::move(err)}; + } + + try { + nlohmann::json data = nlohmann::json::parse(body); + return internal::ParseFullReplicationInfo(data); + } catch (const std::exception& e) { + return {{}, Error{.code = -1, .message = e.what()}}; + } + } + + Error CreateReplication(std::string_view name, ReplicationSettings settings) const noexcept override { + auto json_data = internal::ReplicationSettingsToJsonString(std::move(settings)); + return client_->Post(fmt::format("/replication/{}", name), json_data.dump()); + } + + Error UpdateReplication(std::string_view name, ReplicationSettings settings) const noexcept override { + auto json_data = internal::ReplicationSettingsToJsonString(std::move(settings)); + return client_->Put(fmt::format("/replication/{}", name), json_data.dump()); + } + + Error RemoveReplication(std::string_view name) const noexcept override { + return client_->Delete(fmt::format("/replication/{}", name)); + } + private: HttpOptions options_; std::unique_ptr client_; diff --git a/src/reduct/client.h b/src/reduct/client.h index 9e34270..7977046 100644 --- a/src/reduct/client.h +++ b/src/reduct/client.h @@ -155,15 +155,21 @@ class IClient { */ [[nodiscard]] virtual Result Me() const noexcept = 0; + /** + * Replication information + */ struct ReplicationInfo { - std::string name; // Replication name - bool is_active; // Remote instance is available and replication is active - bool is_provisioned; // Replication settings + std::string name; // Replication name + bool is_active; // Remote instance is available and replication is active + bool is_provisioned; // Replication settings uint64_t pending_records; // Number of records pending replication bool operator<=>(const ReplicationInfo&) const = default; }; + /** + * Replication settings + */ struct ReplicationSettings { std::string src_bucket; // Source bucket std::string dst_bucket; // Destination bucket @@ -177,14 +183,53 @@ class IClient { bool operator<=>(const ReplicationSettings&) const = default; }; - struct ReplicationFullInfo { - ReplicationInfo info; // Replication info + /** + * Replication full info with settings and diagnostics + */ + struct FullReplicationInfo { + ReplicationInfo info; // Replication info ReplicationSettings settings; // Replication settings - Diagnostics diagnostics; // Diagnostics + Diagnostics diagnostics; // Diagnostics - bool operator ==(const ReplicationFullInfo&) const = default; + bool operator==(const FullReplicationInfo&) const = default; }; + /** + * @brief Get list of replications + * @return the list or an error + */ + [[nodiscard]] virtual Result> GetReplicationList() const noexcept = 0; + + /** + * @brief Get replication info with settings and diagnostics + * @param name name of replication + * @return the info or an error + */ + [[nodiscard]] virtual Result GetReplication(std::string_view name) const noexcept = 0; + + /** + * @brief Create a new replication + * @param name name of replication + * @param settings replication settings + * @return error + */ + [[nodiscard]] virtual Error CreateReplication(std::string_view name, ReplicationSettings settings) const noexcept = 0; + + /** + * @brief Update replication settings + * @param name name of replication + * @param settings replication settings + * @return error + */ + [[nodiscard]] virtual Error UpdateReplication(std::string_view name, ReplicationSettings settings) const noexcept = 0; + + + /** + * @brief Remove replication + * @param name name of replication + * @return error + */ + [[nodiscard]] virtual Error RemoveReplication(std::string_view name) const noexcept = 0; /** * @brief Build a client diff --git a/src/reduct/internal/serialisation.cc b/src/reduct/internal/serialisation.cc index a72eb38..c4ec5bf 100644 --- a/src/reduct/internal/serialisation.cc +++ b/src/reduct/internal/serialisation.cc @@ -6,7 +6,7 @@ namespace reduct::internal { -nlohmann::json BucketSettingToJsonString(const IBucket::Settings& settings) noexcept { +nlohmann::json BucketSettingToJsonString(const IBucket::Settings& settings) { nlohmann::json data; const auto& [max_block_size, quota_type, quota_size, max_record_size] = settings; if (max_block_size) { @@ -35,7 +35,7 @@ nlohmann::json BucketSettingToJsonString(const IBucket::Settings& settings) noex return data; } -Result ParseBucketSettings(const nlohmann::json& json) noexcept { +Result ParseBucketSettings(const nlohmann::json& json) { IBucket::Settings settings; try { if (json.contains("max_block_size")) { @@ -63,7 +63,7 @@ Result ParseBucketSettings(const nlohmann::json& json) noexce return {settings, Error::kOk}; } -Result ParseTokenInfo(const nlohmann::json& json) noexcept { +Result ParseTokenInfo(const nlohmann::json& json) { IClient::Time created_at; std::istringstream(json.at("created_at").get()) >> date::parse("%FT%TZ", created_at); @@ -79,4 +79,73 @@ Result ParseTokenInfo(const nlohmann::json& json) noexce Error::kOk, }; } + +Result> ParseReplicationList(const nlohmann::json& data) { + std::vector replication_list; + + auto json_replications = data.at("replications"); + replication_list.reserve(json_replications.size()); + for (const auto& replication : json_replications) { + replication_list.push_back(IClient::ReplicationInfo{ + .name = replication.at("name"), + .is_active = replication.at("is_active"), + .is_provisioned = replication.at("is_provisioned"), + .pending_records = replication.at("pending_records"), + }); + } + + return {replication_list, Error::kOk}; +} + +nlohmann::json ReplicationSettingsToJsonString(IClient::ReplicationSettings settings) { + nlohmann::json json_data; + json_data["src_bucket"] = settings.src_bucket; + json_data["dst_bucket"] = settings.dst_bucket; + json_data["dst_host"] = settings.dst_host; + json_data["dst_token"] = settings.dst_token; + json_data["enabled"] = settings.entries; + json_data["include"] = settings.include; + json_data["exclude"] = settings.exclude; + + return json_data; +} + +Result ParseFullReplicationInfo(const nlohmann::json& data) { + IClient::FullReplicationInfo info; + try { + info.info = IClient::ReplicationInfo{ + .name = data.at("name"), + .is_active = data.at("is_active"), + .is_provisioned = data.at("is_provisioned"), + .pending_records = data.at("pending_records"), + }; + + info.settings = IClient::ReplicationSettings{ + .src_bucket = data.at("src_bucket"), + .dst_bucket = data.at("dst_bucket"), + .dst_host = data.at("dst_host"), + .dst_token = data.at("dst_token"), + .entries = data.at("enabled"), + .include = data.at("include"), + .exclude = data.at("exclude"), + }; + + info.diagnostics = + Diagnostics{.hourly = DiagnosticsItem{.ok = data.at("diagnostics").at("hourly").at("ok"), + .errored = data.at("diagnostics").at("hourly").at("errored"), + .errors = {}}}; + + for (const auto& [key, value] : data.at("diagnostics").at("hourly").at("errors").items()) { + info.diagnostics.hourly.errors[std::stoi(key)] = DiagnosticsError{ + .count = value.at("count"), + .last_message = value.at("last_message"), + }; + } + } catch (const std::exception& ex) { + return {{}, Error{.code = -1, .message = ex.what()}}; + } + + return {info, Error::kOk}; +} + } // namespace reduct::internal diff --git a/src/reduct/internal/serialisation.h b/src/reduct/internal/serialisation.h index 286230b..b3df6f4 100644 --- a/src/reduct/internal/serialisation.h +++ b/src/reduct/internal/serialisation.h @@ -14,21 +14,44 @@ namespace reduct::internal { * @brief Serialize Bucket Setttings to JSON string * @return */ -nlohmann::json BucketSettingToJsonString(const IBucket::Settings& settings) noexcept; +nlohmann::json BucketSettingToJsonString(const IBucket::Settings& settings); /** * @brief Parse Bucket Settings from JSON string * @param json * @return */ -Result ParseBucketSettings(const nlohmann::json& json) noexcept; +Result ParseBucketSettings(const nlohmann::json& json); /** * @brief Parse Bucket Info from JSON string * @param json * @return */ -Result ParseTokenInfo(const nlohmann::json& json) noexcept; +Result ParseTokenInfo(const nlohmann::json& json); + + +/** + * @brief Parse list of replication info from JSON string + * @param json + * @return + */ +Result> ParseReplicationList(const nlohmann::json& data); + + +/** + * @brief Serialize replication settings + * @param settings to serialize + * @return json + */ +nlohmann::json ReplicationSettingsToJsonString(IClient::ReplicationSettings settings); + +/** + * @brief Parse full replication info from JSON string + * @param data + * @return + */ +Result ParseFullReplicationInfo(const nlohmann::json& data); }; // namespace reduct::internal From 13346d6971ef65985e5f00a63360fc63e3e38fa0 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:12:01 +0100 Subject: [PATCH 3/9] clean only test resources --- docs/api_reference/iclient.md | 4 +-- tests/fixture.h | 41 ++++++++++++++++++---------- tests/reduct/bucket_api_test.cc | 4 +-- tests/reduct/entry_api_test.cc | 4 +-- tests/reduct/replication_api_test.cc | 20 ++++++++++++++ tests/reduct/server_api_test.cc | 4 +-- tests/reduct/token_api_test.cc | 26 ++++++++++-------- 7 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 tests/reduct/replication_api_test.cc diff --git a/docs/api_reference/iclient.md b/docs/api_reference/iclient.md index bed3008..32add46 100644 --- a/docs/api_reference/iclient.md +++ b/docs/api_reference/iclient.md @@ -114,8 +114,8 @@ To create a new token, you should use `CreateToken` method with `IClient::Permis ```cpp IClient::Permissions permissions{ .full_access = true, - .read = {"bucket_1"}, - .write = {"bucket_2"}, + .read = {"test_bucket_1"}, + .write = {"test_bucket_2"}, }; diff --git a/tests/fixture.h b/tests/fixture.h index 4ed874e..b7d3c89 100644 --- a/tests/fixture.h +++ b/tests/fixture.h @@ -28,35 +28,46 @@ struct Fixture { throw std::runtime_error(fmt::format("Failed to get bucket list: {}", bucket_list.error.ToString())); } for (auto&& info : client->GetBucketList().result) { + if (!info.name.starts_with("test_bucket")) { + continue; + } std::unique_ptr bucket = client->GetBucket(info.name); [[maybe_unused]] auto ret = bucket->Remove(); } for (auto&& t : client->GetTokenList().result) { - if (t.name != "init-token") { - [[maybe_unused]] auto ret = client->RemoveToken(t.name); + if (!t.name.starts_with("test_token")) { + continue; } + [[maybe_unused]] auto ret = client->RemoveToken(t.name); } - auto [bucket, err] = client->CreateBucket("bucket_1"); + for (auto&& r : client->GetReplicationList().result) { + if (!r.name.starts_with("test_replication")) { + continue; + } + [[maybe_unused]] auto ret = client->RemoveReplication(r.name); + } + + auto [bucket, err] = client->CreateBucket("test_bucket_1"); if (err != reduct::Error::kOk) { - throw std::runtime_error(fmt::format("Failed to create bucket_1: {}", err.ToString())); + throw std::runtime_error(fmt::format("Failed to create test_bucket_1: {}", err.ToString())); } - bucket_1 = std::move(bucket); + test_bucket_1 = std::move(bucket); [[maybe_unused]] auto ret = - bucket_1->Write("entry-1", IBucket::Time() + s(1), [](auto rec) { rec->WriteAll("data-1"); }); - ret = bucket_1->Write("entry-1", IBucket::Time() + s(2), [](auto rec) { rec->WriteAll("data-2"); }); - ret = bucket_1->Write("entry-2", IBucket::Time() + s(3), [](auto rec) { rec->WriteAll("data-3"); }); - ret = bucket_1->Write("entry-2", IBucket::Time() + s(4), [](auto rec) { rec->WriteAll("data-4"); }); - - bucket_2 = client->CreateBucket("bucket_2").result; - ret = bucket_2->Write("entry-1", IBucket::Time() + s(5), [](auto rec) { rec->WriteAll("data-5"); }); - ret = bucket_2->Write("entry-1", IBucket::Time() + s(6), [](auto rec) { rec->WriteAll("data-6"); }); + test_bucket_1->Write("entry-1", IBucket::Time() + s(1), [](auto rec) { rec->WriteAll("data-1"); }); + ret = test_bucket_1->Write("entry-1", IBucket::Time() + s(2), [](auto rec) { rec->WriteAll("data-2"); }); + ret = test_bucket_1->Write("entry-2", IBucket::Time() + s(3), [](auto rec) { rec->WriteAll("data-3"); }); + ret = test_bucket_1->Write("entry-2", IBucket::Time() + s(4), [](auto rec) { rec->WriteAll("data-4"); }); + + test_bucket_2 = client->CreateBucket("test_bucket_2").result; + ret = test_bucket_2->Write("entry-1", IBucket::Time() + s(5), [](auto rec) { rec->WriteAll("data-5"); }); + ret = test_bucket_2->Write("entry-1", IBucket::Time() + s(6), [](auto rec) { rec->WriteAll("data-6"); }); } std::unique_ptr client; - std::unique_ptr bucket_1; - std::unique_ptr bucket_2; + std::unique_ptr test_bucket_1; + std::unique_ptr test_bucket_2; }; #endif // REDUCT_CPP_HELPERS_H diff --git a/tests/reduct/bucket_api_test.cc b/tests/reduct/bucket_api_test.cc index f25af49..216ce7f 100644 --- a/tests/reduct/bucket_api_test.cc +++ b/tests/reduct/bucket_api_test.cc @@ -11,7 +11,7 @@ using reduct::IClient; using s = std::chrono::seconds; -constexpr auto kBucketName = "bucket"; +constexpr auto kBucketName = "test_bucket_3"; TEST_CASE("reduct::Client should create a bucket", "[bucket_api]") { Fixture ctx; @@ -132,7 +132,7 @@ TEST_CASE("reduct::IBucket should get bucket stats", "[bucket_api]") { TEST_CASE("reduct::IBucket should get list of entries", "[bucket_api]") { Fixture ctx; - auto [entries, err] = ctx.bucket_1->GetEntryList(); + auto [entries, err] = ctx.test_bucket_1->GetEntryList(); REQUIRE(err == Error::kOk); REQUIRE(entries.size() == 2); diff --git a/tests/reduct/entry_api_test.cc b/tests/reduct/entry_api_test.cc index 66195ae..a231f59 100644 --- a/tests/reduct/entry_api_test.cc +++ b/tests/reduct/entry_api_test.cc @@ -12,7 +12,7 @@ using reduct::Error; using reduct::IBucket; using reduct::IClient; -const auto kBucketName = "bucket"; +const auto kBucketName = "test_bucket_3"; using us = std::chrono::microseconds; TEST_CASE("reduct::IBucket should write/read a record", "[entry_api]") { @@ -253,7 +253,7 @@ TEST_CASE("reduct::IBucket should query records (huge blobs)", "[entry_api]") { TEST_CASE("reduct::IBucket should limit records in a query", "[entry_api][1_6]") { Fixture ctx; - auto [bucket, _] = ctx.client->GetBucket("bucket_1"); + auto [bucket, _] = ctx.client->GetBucket("test_bucket_1"); REQUIRE(bucket); int count; diff --git a/tests/reduct/replication_api_test.cc b/tests/reduct/replication_api_test.cc new file mode 100644 index 0000000..3d05ae7 --- /dev/null +++ b/tests/reduct/replication_api_test.cc @@ -0,0 +1,20 @@ +// Copyright 2024 Alexey Timin + +#include + +#include + +#include "fixture.h" +#include "reduct/client.h" + +using reduct::Error; +using reduct::IClient; + +TEST_CASE("reduct::Client should get list of replications", "[replication_api]") { + Fixture ctx; + auto [tokens, err] = ctx.client->GetTokenList(); + REQUIRE(err == Error::kOk); + REQUIRE(tokens.size() == 1); + REQUIRE(tokens[0].name == "init-token"); + REQUIRE(tokens[0].created_at.time_since_epoch().count() > 0); +} diff --git a/tests/reduct/server_api_test.cc b/tests/reduct/server_api_test.cc index 0bc8959..cee34be 100644 --- a/tests/reduct/server_api_test.cc +++ b/tests/reduct/server_api_test.cc @@ -38,13 +38,13 @@ TEST_CASE("reduct::Client should list buckets", "[server_api]") { auto [list, err] = ctx.client->GetBucketList(); REQUIRE_FALSE(list.empty()); - REQUIRE(list[0].name == "bucket_1"); + REQUIRE(list[0].name == "test_bucket_1"); REQUIRE(list[0].size == 156); REQUIRE(list[0].entry_count == 2); REQUIRE(list[0].oldest_record.time_since_epoch() == s(1)); REQUIRE(list[0].latest_record.time_since_epoch() == s(4)); - REQUIRE(list[1].name == "bucket_2"); + REQUIRE(list[1].name == "test_bucket_2"); REQUIRE(list[1].size == 78); REQUIRE(list[1].entry_count == 1); REQUIRE(list[1].oldest_record.time_since_epoch() == s(5)); diff --git a/tests/reduct/token_api_test.cc b/tests/reduct/token_api_test.cc index 1aed8c0..b9cb203 100644 --- a/tests/reduct/token_api_test.cc +++ b/tests/reduct/token_api_test.cc @@ -10,6 +10,8 @@ using reduct::Error; using reduct::IClient; +const std::string_view kTestTokenName = "test_token"; + TEST_CASE("reduct::Client should get token list", "[token_api]") { Fixture ctx; auto [tokens, err] = ctx.client->GetTokenList(); @@ -23,16 +25,16 @@ TEST_CASE("reduct::Client should create token", "[token_api]") { Fixture ctx; IClient::Permissions permissions{ .full_access = true, - .read = {"bucket_1"}, - .write = {"bucket_2"}, + .read = {"test_bucket_1"}, + .write = {"test_bucket_2"}, }; - auto [token, err] = ctx.client->CreateToken("test-token", std::move(permissions)); + auto [token, err] = ctx.client->CreateToken(kTestTokenName, std::move(permissions)); REQUIRE(err == Error::kOk); - REQUIRE(token.starts_with("test-token-")); + REQUIRE(token.starts_with("test_token-")); SECTION("Conflict") { - REQUIRE(ctx.client->CreateToken("test-token", {}).error == Error{409, "Token 'test-token' already exists"}); + REQUIRE(ctx.client->CreateToken(kTestTokenName, {}).error == Error{409, "Token 'test_token' already exists"}); } } @@ -40,18 +42,18 @@ TEST_CASE("reduct::Client should get token", "[token_api]") { Fixture ctx; IClient::Permissions permissions{ .full_access = true, - .read = {"bucket_1"}, - .write = {"bucket_2"}, + .read = {"test_bucket_1"}, + .write = {"test_bucket_2"}, }; { - auto [_, err] = ctx.client->CreateToken("test-token", permissions); + auto [_, err] = ctx.client->CreateToken(kTestTokenName, permissions); REQUIRE(err == Error::kOk); } - auto [token, err] = ctx.client->GetToken("test-token"); + auto [token, err] = ctx.client->GetToken(kTestTokenName); REQUIRE(err == Error::kOk); - REQUIRE(token.name == "test-token"); + REQUIRE(token.name == kTestTokenName); REQUIRE(token.created_at.time_since_epoch().count() > 0); REQUIRE(token.is_provisioned == false); REQUIRE(token.permissions.full_access == permissions.full_access); @@ -66,11 +68,11 @@ TEST_CASE("reduct::Client should get token", "[token_api]") { TEST_CASE("reduct::Client should delete token", "[token_api]") { Fixture ctx; { - auto [_, err] = ctx.client->CreateToken("test-token", {}); + auto [_, err] = ctx.client->CreateToken(kTestTokenName, {}); REQUIRE(err == Error::kOk); } - auto err = ctx.client->RemoveToken("test-token"); + auto err = ctx.client->RemoveToken(kTestTokenName); REQUIRE(err == Error::kOk); SECTION("not found") { From 1677e15b8c15a1c42d7e5ecb9ebc8770144e0de7 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:30:44 +0100 Subject: [PATCH 4/9] add tests --- src/reduct/client.cc | 10 ++-- src/reduct/internal/serialisation.cc | 36 +++++++------ tests/CMakeLists.txt | 4 +- tests/reduct/bucket_api_test.cc | 2 +- tests/reduct/entry_api_test.cc | 2 +- tests/reduct/replication_api_test.cc | 80 ++++++++++++++++++++++++++-- tests/reduct/server_api_test.cc | 2 +- tests/reduct/token_api_test.cc | 2 +- 8 files changed, 106 insertions(+), 32 deletions(-) diff --git a/src/reduct/client.cc b/src/reduct/client.cc index 0d04165..d79682a 100644 --- a/src/reduct/client.cc +++ b/src/reduct/client.cc @@ -188,7 +188,7 @@ class Client : public IClient { } Result> GetReplicationList() const noexcept override { - auto [body, err] = client_->Get("/replication"); + auto [body, err] = client_->Get("/replications"); if (err) { return {{}, std::move(err)}; } @@ -202,7 +202,7 @@ class Client : public IClient { } Result GetReplication(std::string_view name) const noexcept override { - auto [body, err] = client_->Get(fmt::format("/replication/{}", name)); + auto [body, err] = client_->Get(fmt::format("/replications/{}", name)); if (err) { return {{}, std::move(err)}; } @@ -217,16 +217,16 @@ class Client : public IClient { Error CreateReplication(std::string_view name, ReplicationSettings settings) const noexcept override { auto json_data = internal::ReplicationSettingsToJsonString(std::move(settings)); - return client_->Post(fmt::format("/replication/{}", name), json_data.dump()); + return client_->Post(fmt::format("/replications/{}", name), json_data.dump()); } Error UpdateReplication(std::string_view name, ReplicationSettings settings) const noexcept override { auto json_data = internal::ReplicationSettingsToJsonString(std::move(settings)); - return client_->Put(fmt::format("/replication/{}", name), json_data.dump()); + return client_->Put(fmt::format("/replications/{}", name), json_data.dump()); } Error RemoveReplication(std::string_view name) const noexcept override { - return client_->Delete(fmt::format("/replication/{}", name)); + return client_->Delete(fmt::format("/replications/{}", name)); } private: diff --git a/src/reduct/internal/serialisation.cc b/src/reduct/internal/serialisation.cc index c4ec5bf..01f19e1 100644 --- a/src/reduct/internal/serialisation.cc +++ b/src/reduct/internal/serialisation.cc @@ -103,7 +103,7 @@ nlohmann::json ReplicationSettingsToJsonString(IClient::ReplicationSettings sett json_data["dst_bucket"] = settings.dst_bucket; json_data["dst_host"] = settings.dst_host; json_data["dst_token"] = settings.dst_token; - json_data["enabled"] = settings.entries; + json_data["entries"] = settings.entries; json_data["include"] = settings.include; json_data["exclude"] = settings.exclude; @@ -114,28 +114,30 @@ Result ParseFullReplicationInfo(const nlohmann::js IClient::FullReplicationInfo info; try { info.info = IClient::ReplicationInfo{ - .name = data.at("name"), - .is_active = data.at("is_active"), - .is_provisioned = data.at("is_provisioned"), - .pending_records = data.at("pending_records"), + .name = data.at("info").at("name"), + .is_active = data.at("info").at("is_active"), + .is_provisioned = data.at("info").at("is_provisioned"), + .pending_records = data.at("info").at("pending_records"), }; info.settings = IClient::ReplicationSettings{ - .src_bucket = data.at("src_bucket"), - .dst_bucket = data.at("dst_bucket"), - .dst_host = data.at("dst_host"), - .dst_token = data.at("dst_token"), - .entries = data.at("enabled"), - .include = data.at("include"), - .exclude = data.at("exclude"), + .src_bucket = data.at("settings").at("src_bucket"), + .dst_bucket = data.at("settings").at("dst_bucket"), + .dst_host = data.at("settings").at("dst_host"), + .dst_token = data.at("settings").at("dst_token"), + .entries = data.at("settings").at("entries"), + .include = data.at("settings").at("include"), + .exclude = data.at("settings").at("exclude"), }; - info.diagnostics = - Diagnostics{.hourly = DiagnosticsItem{.ok = data.at("diagnostics").at("hourly").at("ok"), - .errored = data.at("diagnostics").at("hourly").at("errored"), - .errors = {}}}; + auto diagnostics = data.at("diagnostics"); + info.diagnostics = Diagnostics{.hourly = DiagnosticsItem{ + .ok = diagnostics.at("hourly").at("ok"), + .errored = diagnostics.at("hourly").at("errored"), + .errors = {}, + }}; - for (const auto& [key, value] : data.at("diagnostics").at("hourly").at("errors").items()) { + for (const auto& [key, value] : diagnostics.at("hourly").at("errors").items()) { info.diagnostics.hourly.errors[std::stoi(key)] = DiagnosticsError{ .count = value.at("count"), .last_message = value.at("last_message"), diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7ad43d7..156c755 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,8 +15,10 @@ FetchContent_MakeAvailable(Catch2) set(SRC_FILES reduct/bucket_api_test.cc reduct/entry_api_test.cc + reduct/replication_api_test.cc reduct/server_api_test.cc - test.cc reduct/token_api_test.cc) + reduct/token_api_test.cc + test.cc) add_executable(reduct-tests ${SRC_FILES}) diff --git a/tests/reduct/bucket_api_test.cc b/tests/reduct/bucket_api_test.cc index 216ce7f..7d53563 100644 --- a/tests/reduct/bucket_api_test.cc +++ b/tests/reduct/bucket_api_test.cc @@ -1,4 +1,4 @@ -// Copyright 2022 Alexey Timin +// Copyright 2022-2024 Alexey Timin #include diff --git a/tests/reduct/entry_api_test.cc b/tests/reduct/entry_api_test.cc index a231f59..ca22abd 100644 --- a/tests/reduct/entry_api_test.cc +++ b/tests/reduct/entry_api_test.cc @@ -1,4 +1,4 @@ -// Copyright 2022 Alexey Timin +// Copyright 2022-2024 Alexey Timin #include diff --git a/tests/reduct/replication_api_test.cc b/tests/reduct/replication_api_test.cc index 3d05ae7..51517d8 100644 --- a/tests/reduct/replication_api_test.cc +++ b/tests/reduct/replication_api_test.cc @@ -1,4 +1,4 @@ -// Copyright 2024 Alexey Timin +// Copyright 2022-2024 Alexey Timin #include @@ -10,11 +10,81 @@ using reduct::Error; using reduct::IClient; +IClient::ReplicationSettings settings{ + .src_bucket = "test_bucket_1", + .dst_bucket = "test_bucket_2", + .dst_host = "http://127.0.0.1:8383", + .entries = {"entry-1"}, + .include = {{"label-3", "value-4"}}, + .exclude = {{"label-1", "value-2"}}, +}; + TEST_CASE("reduct::Client should get list of replications", "[replication_api]") { Fixture ctx; - auto [tokens, err] = ctx.client->GetTokenList(); + auto [replications, err] = ctx.client->GetReplicationList(); + REQUIRE(err == Error::kOk); + REQUIRE(replications.size() == 0); +} + +TEST_CASE("reduct::Client should create a replication", "[replication_api]") { + Fixture ctx; + + auto err = ctx.client->CreateReplication("test_replication", settings); + REQUIRE(err == Error::kOk); + + auto [replication, err_2] = ctx.client->GetReplication("test_replication"); + REQUIRE(err_2 == Error::kOk); + REQUIRE(replication.info == IClient::ReplicationInfo{ + .name = "test_replication", + .is_active = false, + .is_provisioned = false, + .pending_records = 0, + }); + + settings.dst_token = "***"; + REQUIRE(replication.settings == settings); + REQUIRE(replication.diagnostics == reduct::Diagnostics{}); + + SECTION("Conflict") { + REQUIRE(ctx.client->CreateReplication("test_replication", {}) == + Error{409, "Replication 'test_replication' already exists"}); + } +} + +TEST_CASE("reduct::Client should update a replication", "[replication_api]") { + Fixture ctx; + auto err = ctx.client->CreateReplication("test_replication", settings); REQUIRE(err == Error::kOk); - REQUIRE(tokens.size() == 1); - REQUIRE(tokens[0].name == "init-token"); - REQUIRE(tokens[0].created_at.time_since_epoch().count() > 0); + + settings.entries = {"entry-2"}; + err = ctx.client->UpdateReplication("test_replication", settings); + REQUIRE(err == Error::kOk); + + auto [replication, err_2] = ctx.client->GetReplication("test_replication"); + REQUIRE(err_2 == Error::kOk); + + settings.dst_token = "***"; + REQUIRE(replication.settings == settings); + + SECTION("Not found") { + REQUIRE(ctx.client->UpdateReplication("test_replication_2", {}) == + Error{404, "Replication 'test_replication_2' does not exist"}); + } +} + +TEST_CASE("reduct::Client should remove a replication", "[replication_api]") { + Fixture ctx; + auto err = ctx.client->CreateReplication("test_replication", settings); + REQUIRE(err == Error::kOk); + + err = ctx.client->RemoveReplication("test_replication"); + REQUIRE(err == Error::kOk); + + auto [replication, err_2] = ctx.client->GetReplication("test_replication"); + REQUIRE(err_2 == Error{404, "Replication 'test_replication' does not exist"}); + + SECTION("Not found") { + REQUIRE(ctx.client->RemoveReplication("test_replication_2") == + Error{404, "Replication 'test_replication_2' does not exist"}); + } } diff --git a/tests/reduct/server_api_test.cc b/tests/reduct/server_api_test.cc index cee34be..b7055e8 100644 --- a/tests/reduct/server_api_test.cc +++ b/tests/reduct/server_api_test.cc @@ -1,4 +1,4 @@ -// Copyright 2022 Alexey Timin +// Copyright 2022-2024 Alexey Timin #include diff --git a/tests/reduct/token_api_test.cc b/tests/reduct/token_api_test.cc index b9cb203..21afe8d 100644 --- a/tests/reduct/token_api_test.cc +++ b/tests/reduct/token_api_test.cc @@ -1,4 +1,4 @@ -// Copyright 2022 Alexey Timin +// Copyright 2022-2024 Alexey Timin #include From a38191adeafd0288c35e92dd306c71ea2a043e8c Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:33:41 +0100 Subject: [PATCH 5/9] bump version and update dependencies --- CMakeLists.txt | 4 ++-- conanfile.py | 12 ++++++------ examples/CMakeLists.txt | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f4a0ea2..9993abd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.18) set(MAJOR_VERSION 1) -set(MINOR_VERSION 7) -set(PATCH_VERSION 1) +set(MINOR_VERSION 8) +set(PATCH_VERSION 0) set(REDUCT_CPP_FULL_VERSION ${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}) project(reductcpp VERSION ${REDUCT_CPP_FULL_VERSION}) diff --git a/conanfile.py b/conanfile.py index 70e144c..556454e 100644 --- a/conanfile.py +++ b/conanfile.py @@ -3,7 +3,7 @@ class DriftFrameworkConan(ConanFile): name = "reduct-cpp" - version = "1.7.1" + version = "1.8.0" license = "MIT" author = "Alexey Timin" url = "https://github.com/reduct-storage/reduct-cpp" @@ -15,11 +15,11 @@ class DriftFrameworkConan(ConanFile): "date:header_only": True} generators = "cmake" - requires = ("fmt/10.0.0", - "cpp-httplib/0.12.4", - "nlohmann_json/3.11.2", - "openssl/3.1.1", - "concurrentqueue/1.0.3", + requires = ("fmt/10.2.1", + "cpp-httplib/0.14.3", + "nlohmann_json/3.11.3", + "openssl/3.2.0", + "concurrentqueue/1.0.4", "date/3.0.1") def config_options(self): diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 26da486..cd871d0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -8,7 +8,7 @@ find_package(OpenSSL) -find_package(ReductCpp 1.7.0) +find_package(ReductCpp 1.8.0) add_executable(usage-example usage_example.cc) add_executable(subscription subscription.cc) From 77709b2739884a32df8d52de136d8dba3362f7e5 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:34:46 +0100 Subject: [PATCH 6/9] mark replication tests for 1.8 --- tests/reduct/replication_api_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/reduct/replication_api_test.cc b/tests/reduct/replication_api_test.cc index 51517d8..2a2a526 100644 --- a/tests/reduct/replication_api_test.cc +++ b/tests/reduct/replication_api_test.cc @@ -19,14 +19,14 @@ IClient::ReplicationSettings settings{ .exclude = {{"label-1", "value-2"}}, }; -TEST_CASE("reduct::Client should get list of replications", "[replication_api]") { +TEST_CASE("reduct::Client should get list of replications", "[replication_api][1_8]") { Fixture ctx; auto [replications, err] = ctx.client->GetReplicationList(); REQUIRE(err == Error::kOk); REQUIRE(replications.size() == 0); } -TEST_CASE("reduct::Client should create a replication", "[replication_api]") { +TEST_CASE("reduct::Client should create a replication", "[replication_api][1_8]") { Fixture ctx; auto err = ctx.client->CreateReplication("test_replication", settings); @@ -51,7 +51,7 @@ TEST_CASE("reduct::Client should create a replication", "[replication_api]") { } } -TEST_CASE("reduct::Client should update a replication", "[replication_api]") { +TEST_CASE("reduct::Client should update a replication", "[replication_api][1_8]") { Fixture ctx; auto err = ctx.client->CreateReplication("test_replication", settings); REQUIRE(err == Error::kOk); @@ -72,7 +72,7 @@ TEST_CASE("reduct::Client should update a replication", "[replication_api]") { } } -TEST_CASE("reduct::Client should remove a replication", "[replication_api]") { +TEST_CASE("reduct::Client should remove a replication", "[replication_api][1_8]") { Fixture ctx; auto err = ctx.client->CreateReplication("test_replication", settings); REQUIRE(err == Error::kOk); From 8532871d733418cd64847027e64568f25fc25ec0 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:42:09 +0100 Subject: [PATCH 7/9] update fetch content deps --- cmake/InstallDependencies.cmake | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmake/InstallDependencies.cmake b/cmake/InstallDependencies.cmake index 7fa2555..af91360 100644 --- a/cmake/InstallDependencies.cmake +++ b/cmake/InstallDependencies.cmake @@ -27,26 +27,26 @@ else () include(FetchContent) FetchContent_Declare( fmt - URL https://github.com/fmtlib/fmt/archive/refs/tags/8.1.1.zip - URL_HASH MD5=fed2f2c5027a4034cc8351bf59aa8f7c + URL https://github.com/fmtlib/fmt/archive/refs/tags/10.2.1.zip + URL_HASH MD5=1bba4e8bdd7b0fa98f207559ffa380a3 ) FetchContent_Declare( nlohmann_json - URL https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.zip - URL_HASH MD5=accaeb6a75f5972f479ef9139fa65b9e + URL https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.zip + URL_HASH MD5=23712ebf3a4b4ccb39f2375521716ab3 ) FetchContent_Declare( httplib - URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.10.7.zip - URL_HASH MD5=31497d5f3ff1e0df2f57195dbabd3198 + URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.14.3.zip + URL_HASH MD5=af82eb38506ca531b6d1d53524ff7912 ) FetchContent_Declare( concurrentqueue - URL https://github.com/cameron314/concurrentqueue/archive/refs/tags/v1.0.3.zip - URL_HASH MD5=6e879b14c833df7c011be5959e70cef7 + URL https://github.com/cameron314/concurrentqueue/archive/refs/tags/v1.0.4.zip + URL_HASH MD5=814c5e121b29e37ee836312f0eb0328f ) FetchContent_Declare( From 27f038183164913e05b98bca00b7efabd4dc4d98 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:43:41 +0100 Subject: [PATCH 8/9] exclude 1.8 tests for latest version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9821c24..339f8c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: - reductstore_version: "main" exclude_api_version_tag: "" - reductstore_version: "latest" - exclude_api_version_tag: "~[1_7]" + exclude_api_version_tag: "~[1_8]" needs: - build From 9cc5ef51fd2d53af9301cd53776f6ec875b61f04 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 23 Jan 2024 23:45:06 +0100 Subject: [PATCH 9/9] update CHANGELOG and README --- CHANGELOG.md | 5 +++++ README.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dccdb59..1bca873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + +### Added + +- RS-44: Replication API, [PR-64](https://github.com/reductstore/reduct-cpp/pull/64) + ### Changed: - docs: update link to new website, [PR-63](https://github.com/reductstore/reduct-cpp/pull/63) diff --git a/README.md b/README.md index 19ba248..d0aca39 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ in C++20. It allows developers to easily interact with the database from their C ## Features * Written in C++20 -* Support ReductStore HTTP API v1.7 +* Support ReductStore HTTP API v1.8 * Support HTTP and HTTPS protocols * Exception free * Support Linux AMD64