Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

disable meta upgrade from v1 to v3 directly #3798

Merged
merged 1 commit into from
Jan 26, 2022
Merged

disable meta upgrade from v1 to v3 directly #3798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/daemons/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ nebula_add_executable(
$<TARGET_OBJECTS:meta_version_man_obj>
$<TARGET_OBJECTS:meta_data_upgrade_obj>
$<TARGET_OBJECTS:meta_v2_thrift_obj>
$<TARGET_OBJECTS:meta_v1_thrift_obj>
${common_deps}
${storage_meta_deps}
LIBRARIES
Expand Down Expand Up @@ -243,7 +242,6 @@ nebula_add_executable(
$<TARGET_OBJECTS:meta_http_handler>
$<TARGET_OBJECTS:meta_version_man_obj>
$<TARGET_OBJECTS:meta_data_upgrade_obj>
$<TARGET_OBJECTS:meta_v1_thrift_obj>
${storage_meta_deps}
${common_deps}
LIBRARIES
Expand Down
9 changes: 2 additions & 7 deletions src/daemons/MetaDaemonInit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,13 +145,8 @@ std::unique_ptr<nebula::kvstore::KVStore> initKV(std::vector<nebula::HostAddr> p
LOG(ERROR) << "Meta version is invalid";
return nullptr;
} else if (version == nebula::meta::MetaVersion::V1) {
auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(engine);
if (!ret.ok()) {
LOG(ERROR) << "Update meta from V1 to V2 failed " << ret;
return nullptr;
}

nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V2);
LOG(ERROR) << "Can't upgrade meta from V1 to V3";
return nullptr;
} else if (version == nebula::meta::MetaVersion::V2) {
critical27 marked this conversation as resolved.
Show resolved Hide resolved
auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(engine);
if (!ret.ok()) {
Expand Down
2 changes: 0 additions & 2 deletions src/meta/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ nebula_add_library(

add_dependencies(
meta_version_man_obj
meta_v1_thrift_obj
meta_v2_thrift_obj
)

Expand All @@ -116,7 +115,6 @@ set(meta_test_deps
$<TARGET_OBJECTS:meta_service_handler>
$<TARGET_OBJECTS:meta_version_man_obj>
$<TARGET_OBJECTS:meta_v2_thrift_obj>
$<TARGET_OBJECTS:meta_v1_thrift_obj>
$<TARGET_OBJECTS:meta_data_upgrade_obj>
$<TARGET_OBJECTS:storage_admin_service_handler>
$<TARGET_OBJECTS:graph_storage_service_handler>
Expand Down
277 changes: 0 additions & 277 deletions src/meta/MetaVersionMan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
#include "meta/processors/job/JobDescription.h"
#include "meta/processors/job/JobUtils.h"
#include "meta/upgrade/MetaDataUpgrade.h"
#include "meta/upgrade/v1/MetaServiceUtilsV1.h"
#include "meta/upgrade/v2/MetaServiceUtilsV2.h"

DEFINE_bool(null_type, true, "set schema to support null type");
Expand Down Expand Up @@ -60,39 +59,6 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion v
return code == nebula::cpp2::ErrorCode::SUCCEEDED;
}

// static
Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVEngine* engine) {
CHECK_NOTNULL(engine);
auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr());

std::string path = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot);
if (!fs::FileUtils::exist(path) && !fs::FileUtils::makeDir(path)) {
LOG(ERROR) << "Make checkpoint dir: " << path << " failed";
return Status::Error("Create snapshot file failed");
}

std::string dataPath = folly::sformat("{}/data", path);
auto code = engine->createCheckpoint(dataPath);
if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
LOG(ERROR) << "Create snapshot failed: " << snapshot;
return Status::Error("Create snapshot failed");
}

auto status = doUpgradeV1ToV2(engine);
if (!status.ok()) {
// rollback by snapshot
return status;
}

// delete snapshot file
auto checkpointPath = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot);

if (fs::FileUtils::exist(checkpointPath) && !fs::FileUtils::remove(checkpointPath.data(), true)) {
LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually";
}
return Status::OK();
}

Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
CHECK_NOTNULL(engine);
auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr());
Expand Down Expand Up @@ -124,249 +90,6 @@ Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
return Status::OK();
}

// static
Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVEngine* engine) {
MetaDataUpgrade upgrader(engine);
{
// kSpacesTable
auto prefix = nebula::meta::v1::kSpacesTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printSpacesV1(iter->val());
}
status = upgrader.rewriteSpaces(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kPartsTable
auto prefix = nebula::meta::v1::kPartsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printParts(iter->key(), iter->val());
}
status = upgrader.rewriteParts(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kHostsTable
auto prefix = nebula::meta::v1::kHostsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printHost(iter->key(), iter->val());
}
status = upgrader.rewriteHosts(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kLeadersTable
auto prefix = nebula::meta::v1::kLeadersTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printLeaders(iter->key());
}
status = upgrader.rewriteLeaders(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kTagsTable
auto prefix = nebula::meta::v1::kTagsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printSchemas(iter->val());
}
status = upgrader.rewriteSchemas(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kEdgesTable
auto prefix = nebula::meta::v1::kEdgesTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printSchemas(iter->val());
}
status = upgrader.rewriteSchemas(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kIndexesTable
auto prefix = nebula::meta::v1::kIndexesTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printIndexes(iter->val());
}
status = upgrader.rewriteIndexes(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kConfigsTable
auto prefix = nebula::meta::v1::kConfigsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printConfigs(iter->key(), iter->val());
}
status = upgrader.rewriteConfigs(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kJob
auto prefix = JobUtil::jobPrefix();
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (JobDescription::isJobKey(iter->key())) {
if (FLAGS_print_info) {
upgrader.printJobDesc(iter->key(), iter->val());
}
status = upgrader.rewriteJobDesc(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
} else {
// the job key format is change, need to delete the old format
status = upgrader.deleteKeyVal(iter->key());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
}
iter->next();
}
}
}

// delete
{
std::vector<std::string> prefixes({nebula::meta::v1::kIndexStatusTable,
nebula::meta::v1::kDefaultTable,
nebula::meta::v1::kCurrJob,
nebula::meta::v1::kJobArchive});
std::unique_ptr<kvstore::KVIterator> iter;
for (auto& prefix : prefixes) {
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (prefix == nebula::meta::v1::kIndexesTable) {
if (FLAGS_print_info) {
upgrader.printIndexes(iter->val());
}
auto oldItem = meta::v1::MetaServiceUtilsV1::parseIndex(iter->val());
auto spaceId = MetaKeyUtils::parseIndexesKeySpaceID(iter->key());
status = upgrader.deleteKeyVal(
MetaKeyUtils::indexIndexKey(spaceId, oldItem.get_index_name()));
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
}
status = upgrader.deleteKeyVal(iter->key());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}
}
if (!setMetaVersionToKV(engine, MetaVersion::V2)) {
return Status::Error("Persist meta version failed");
} else {
return Status::OK();
}
}

Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVEngine* engine) {
MetaDataUpgrade upgrader(engine);
// Step 1: Upgrade HeartBeat into machine list
Expand Down
4 changes: 0 additions & 4 deletions src/meta/MetaVersionMan.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,11 @@ class MetaVersionMan final {

static bool setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion version);

static Status updateMetaV1ToV2(kvstore::KVEngine* engine);

static Status updateMetaV2ToV3(kvstore::KVEngine* engine);

private:
static MetaVersion getVersionByHost(kvstore::KVStore* kv);

static Status doUpgradeV1ToV2(kvstore::KVEngine* engine);

static Status doUpgradeV2ToV3(kvstore::KVEngine* engine);
};

Expand Down
3 changes: 0 additions & 3 deletions src/meta/upgrade/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,12 @@
nebula_add_library(
meta_data_upgrade_obj OBJECT
MetaDataUpgrade.cpp
v1/MetaServiceUtilsV1.cpp
v2/MetaServiceUtilsV2.cpp
)

add_dependencies(
meta_data_upgrade_obj
meta_v1_thrift_obj
meta_v2_thrift_obj
)

nebula_add_subdirectory(v1)
nebula_add_subdirectory(v2)
Loading