Skip to content

Commit

Permalink
[#7126] Add restore_snapshot_schedule to admin
Browse files Browse the repository at this point in the history 
Summary:
This diff adds admin command restore_snapshot_schedule to restore snapshot schedule at specified time.
Also added command list_snapshot_restorations to list snapshot restorations.

Test Plan: ybd --cxx-test yb-admin-test --gtest_filter AdminCliTest.SnapshotSchedule

Reviewers: bogdan, oleg

Reviewed By: oleg

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D11177
  • Loading branch information
@spolitov
spolitov committed Apr 12, 2021
1 parent df5adb3 commit 41a5225
Show file tree
Hide file tree
Showing 20 changed files with 549 additions and 296 deletions.
11 changes: 8 additions & 3 deletions ent/src/yb/master/catalog_manager_ent.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,14 @@ Status CatalogManager::CreateSnapshot(const CreateSnapshotRequestPB* req,
return CreateTransactionAwareSnapshot(*req, resp, rpc);
}

if (req->has_schedule_id()) {
auto schedule_id = VERIFY_RESULT(FullyDecodeSnapshotScheduleId(req->schedule_id()));
auto snapshot_id = VERIFY_RESULT(
snapshot_coordinator_.CreateForSchedule(schedule_id, rpc->GetClientDeadline()));
resp->set_snapshot_id(snapshot_id.data(), snapshot_id.size());
return Status::OK();
}

return CreateNonTransactionAwareSnapshot(req, resp, rpc);
}

Expand Down Expand Up @@ -489,9 +497,6 @@ Status CatalogManager::RestoreSnapshot(const RestoreSnapshotRequestPB* req,
HybridTime ht;
if (req->has_restore_ht()) {
ht = HybridTime(req->restore_ht());
} else if (req->has_restore_interval()) {
ht = HybridTime::FromMicros(
master_->clock()->Now().GetPhysicalValueMicros() - req->restore_interval());
}
TxnSnapshotRestorationId id = VERIFY_RESULT(snapshot_coordinator_.Restore(txn_snapshot_id, ht));
resp->set_restoration_id(id.data(), id.size());
Expand Down
22 changes: 20 additions & 2 deletions ent/src/yb/master/master_backup.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ message CreateSnapshotRequestPB {
// Interpet this snapshot as imported:
// Snapshot will be created as COMPLETE and no tablet operations will be performed.
optional bool imported = 4;

optional bytes schedule_id = 5; // Create snapshot to this schedule. Other fields are ignored.
}

message CreateSnapshotResponsePB {
Expand All @@ -85,6 +87,23 @@ message SnapshotInfoPB {
optional SysSnapshotEntryPB entry = 2;
}

message TabletRestorationPB {
optional bytes id = 1;
optional SysSnapshotEntryPB.State state = 2;
}

message SysRestorationEntryPB {
optional SysSnapshotEntryPB.State state = 1 [ default = UNKNOWN ];
repeated TabletRestorationPB tablet_restorations = 2;
optional bytes snapshot_id = 8;
}

message RestorationInfoPB {
optional bytes id = 1;
// Detailed snapshot entries.
optional SysRestorationEntryPB entry = 2;
}

message ListSnapshotsRequestPB {
optional bytes snapshot_id = 2;
optional bool list_deleted_snapshots = 3;
Expand All @@ -100,7 +119,6 @@ message ListSnapshotsResponsePB {
message RestoreSnapshotRequestPB {
optional bytes snapshot_id = 2;
optional fixed64 restore_ht = 3; // Absolute Timing Option: Max HybridTime, in Micros.
optional fixed64 restore_interval = 4; // Relative Timing Option: Micros before Now to rewind.
}

message RestoreSnapshotResponsePB {
Expand Down Expand Up @@ -128,7 +146,7 @@ message ListSnapshotRestorationsRequestPB {
message ListSnapshotRestorationsResponsePB {
optional AppStatusPB status = 1;

repeated SnapshotInfoPB restorations = 2;
repeated RestorationInfoPB restorations = 2;
}

// ID mapping pair: old ID (object ID on external cluster) TO new ID
Expand Down
195 changes: 101 additions & 94 deletions ent/src/yb/tools/yb-admin-test_ent.cc
View file

Large diffs are not rendered by default.

88 changes: 79 additions & 9 deletions ent/src/yb/tools/yb-admin_cli_ent.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@
#include "yb/tools/yb-admin_cli.h"

#include <iostream>
#include <regex>

#include <boost/algorithm/string.hpp>

#include "yb/common/snapshot.h"
#include "yb/tools/yb-admin_client.h"
#include "yb/util/date_time.h"
#include "yb/util/stol_utils.h"
#include "yb/util/string_case.h"
#include "yb/util/tostring.h"
Expand All @@ -34,6 +36,46 @@ using std::vector;
using client::YBTableName;
using strings::Substitute;

namespace {

Result<HybridTime> ParseHybridTime(const string& timestamp) {
// Acceptable system time formats:
// 1. HybridTime Timestamp (in Microseconds)
// 2. -Interval
// 3. Human readable string
auto ts = boost::trim_copy(timestamp);

HybridTime ht;
// The HybridTime is given in milliseconds and will contain 16 chars.
static const std::regex int_regex("[0-9]{16}");
if (std::regex_match(ts, int_regex)) {
return HybridTime::FromMicros(std::stoul(ts));
}
if (!ts.empty() && ts[0] == '-') {
return HybridTime::FromMicros(
VERIFY_RESULT(WallClock()->Now()).time_point -
VERIFY_RESULT(DateTime::IntervalFromString(ts.substr(1))).ToMicroseconds());
}
return HybridTime::FromMicros(VERIFY_RESULT(DateTime::TimestampFromString(ts)).ToInt64());;
}

const string kMinus = "minus";

template <class T, class Args>
Result<T> GetOptionalArg(const Args& args, size_t idx) {
if (args.size() <= idx) {
return T::Nil();
}
if (args.size() > idx + 1) {
return STATUS_FORMAT(InvalidArgument,
"Too many arguments for command, at most $0 expected, but $1 found",
idx + 1, args.size());
}
return VERIFY_RESULT(T::FromString(args[idx]));
}

} // namespace

void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) {
super::RegisterCommandHandlers(client);

Expand Down Expand Up @@ -89,6 +131,14 @@ void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) {
return Status::OK();
});

RegisterJson(
"list_snapshot_restorations",
" [<restoration_id>]",
[client](const CLIArguments& args) -> Result<rapidjson::Document> {
auto restoration_id = VERIFY_RESULT(GetOptionalArg<TxnSnapshotRestorationId>(args, 0));
return client->ListSnapshotRestorations(restoration_id);
});

RegisterJson(
"create_snapshot_schedule",
" <snapshot_interval_in_minutes>"
Expand All @@ -110,12 +160,29 @@ void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) {
"list_snapshot_schedules",
" [<schedule_id>]",
[client](const CLIArguments& args) -> Result<rapidjson::Document> {
if (args.size() > 1) {
auto schedule_id = VERIFY_RESULT(GetOptionalArg<SnapshotScheduleId>(args, 0));
return client->ListSnapshotSchedules(schedule_id);
});

RegisterJson(
"restore_snapshot_schedule",
Format(" <schedule_id> (<timestamp> | $0 <interval>)", kMinus),
[client](const CLIArguments& args) -> Result<rapidjson::Document> {
if (args.size() < 2 || args.size() > 3) {
return ClusterAdminCli::kInvalidArguments;
}
auto schedule_id = args.empty() ? SnapshotScheduleId::Nil()
: VERIFY_RESULT(SnapshotScheduleId::FromString(args[0]));
return client->ListSnapshotSchedules(schedule_id);
auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0]));
HybridTime restore_at;
if (args.size() == 2) {
restore_at = VERIFY_RESULT(ParseHybridTime(args[1]));
} else {
if (args[1] != kMinus) {
return ClusterAdminCli::kInvalidArguments;
}
restore_at = VERIFY_RESULT(ParseHybridTime("-" + args[2]));
}

return client->RestoreSnapshotSchedule(schedule_id, restore_at);
});

Register(
Expand Down Expand Up @@ -156,18 +223,21 @@ void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) {
});

Register(
"restore_snapshot", " <snapshot_id> [{<timestamp> | minus {interval}]",
"restore_snapshot", Format(" <snapshot_id> [{<timestamp> | $0 {interval}]", kMinus),
[client](const CLIArguments& args) -> Status {
if (args.size() < 1 || 3 < args.size()) {
return ClusterAdminCli::kInvalidArguments;
} else if (args.size() == 3 && args[1] != "minus") {
} else if (args.size() == 3 && args[1] != kMinus) {
return ClusterAdminCli::kInvalidArguments;
}
const string snapshot_id = args[0];
string timestamp = (args.size() == 2) ? args[1] : "";
if (args.size() == 3) {
timestamp = "-" + args[2];
HybridTime timestamp;
if (args.size() == 2) {
timestamp = VERIFY_RESULT(ParseHybridTime(args[1]));
} else if (args.size() == 3) {
timestamp = VERIFY_RESULT(ParseHybridTime("-" + args[2]));
}

RETURN_NOT_OK_PREPEND(client->RestoreSnapshot(snapshot_id, timestamp),
Substitute("Unable to restore snapshot $0", snapshot_id));
return Status::OK();
Expand Down
6 changes: 5 additions & 1 deletion ent/src/yb/tools/yb-admin_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,15 @@ class ClusterAdminClient : public yb::tools::ClusterAdminClient {
const bool add_indexes = true,
const int flush_timeout_secs = 0);
CHECKED_STATUS CreateNamespaceSnapshot(const TypedNamespaceName& ns);
Result<rapidjson::Document> ListSnapshotRestorations(
const TxnSnapshotRestorationId& restoration_id);
Result<rapidjson::Document> CreateSnapshotSchedule(const std::vector<client::YBTableName>& tables,
MonoDelta interval, MonoDelta retention);
Result<rapidjson::Document> ListSnapshotSchedules(const SnapshotScheduleId& schedule_id);
Result<rapidjson::Document> RestoreSnapshotSchedule(
const SnapshotScheduleId& schedule_id, HybridTime restore_at);
CHECKED_STATUS RestoreSnapshot(const std::string& snapshot_id,
const std::string& timestamp);
HybridTime timestamp);
CHECKED_STATUS DeleteSnapshot(const std::string& snapshot_id);

CHECKED_STATUS CreateSnapshotMetaFile(const std::string& snapshot_id,
Expand Down
Loading

0 comments on commit 41a5225

Please sign in to comment.