diff --git a/cmake/nebula/CompilerLauncher.cmake b/cmake/nebula/CompilerLauncher.cmake index 2295e857934..1c8f12f7cb4 100644 --- a/cmake/nebula/CompilerLauncher.cmake +++ b/cmake/nebula/CompilerLauncher.cmake @@ -1,5 +1,5 @@ option(ENABLE_COMPILER_LAUNCHER "Using compiler launcher if available" ON) -if(NOT ENABLE_COMPILER_LAUNCHER) +if(NOT (ENABLE_COMPILER_LAUNCHER AND ENABLE_CCACHE)) return() endif() diff --git a/docker/README.md b/docker/README.md index 6cd7391c46e..92365b474ac 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,4 +5,4 @@ Following docker images will be ready in production. - [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd` - [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad` - [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged` -- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader +- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump and meta_dump diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 86ffb774c33..ce0e7cfc381 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -737,7 +737,7 @@ class MetaClient : public BaseMetaClient { ListenersMap doGetListenersMap(const HostAddr& host, const LocalCache& localCache); - // Checks if the the client version is compatible with the server version by checking the + // Checks if the client version is compatible with the server version by checking the // whilelist in meta. Status verifyVersion(); diff --git a/src/common/conf/test/ConfigurationTest.cpp b/src/common/conf/test/ConfigurationTest.cpp index 6ed00c7d145..daa18d39c35 100644 --- a/src/common/conf/test/ConfigurationTest.cpp +++ b/src/common/conf/test/ConfigurationTest.cpp @@ -123,7 +123,7 @@ TEST(Configuration, ParseFromFile) { fprintf(file, "// This a C++ style comment line\n"); fprintf(file, "# This a shell style comment line\n"); fprintf(file, "{ \n"); - fprintf(file, "\t\"int\": 123, // this is a integer\n"); + fprintf(file, "\t\"int\": 123, // this is an integer\n"); fprintf(file, "\t\"double\": 3.14 # this is a double\n"); fprintf(file, "} \n"); ::fflush(file); diff --git a/src/common/utils/NebulaKeyUtils.cpp b/src/common/utils/NebulaKeyUtils.cpp index 479ad2c939c..d026f3efed0 100644 --- a/src/common/utils/NebulaKeyUtils.cpp +++ b/src/common/utils/NebulaKeyUtils.cpp @@ -253,7 +253,6 @@ std::vector NebulaKeyUtils::snapshotPrefix(PartitionID partId) { if (partId == 0) { result.emplace_back(""); } else { - result.emplace_back(vertexPrefix(partId)); result.emplace_back(tagPrefix(partId)); result.emplace_back(edgePrefix(partId)); result.emplace_back(IndexKeyUtils::indexPrefix(partId)); diff --git a/src/common/utils/NebulaKeyUtils.h b/src/common/utils/NebulaKeyUtils.h index 6608c4d5bdf..68c0c0d8f8a 100644 --- a/src/common/utils/NebulaKeyUtils.h +++ b/src/common/utils/NebulaKeyUtils.h @@ -155,6 +155,12 @@ class NebulaKeyUtils final { return static_cast(type) == NebulaKeyType::kEdge; } + static bool isVertex(const folly::StringPiece& rawKey) { + constexpr int32_t len = static_cast(sizeof(NebulaKeyType)); + auto type = readInt(rawKey.data(), len) & kTypeMask; + return static_cast(type) == NebulaKeyType::kVertex; + } + static bool isLock(size_t vIdLen, const folly::StringPiece& rawKey) { return isEdge(vIdLen, rawKey, kLockVersion); } diff --git a/src/daemons/MetaDaemon.cpp b/src/daemons/MetaDaemon.cpp index 5ccedd0f1ff..0fa5f7ebafc 100644 --- a/src/daemons/MetaDaemon.cpp +++ b/src/daemons/MetaDaemon.cpp @@ -7,6 +7,7 @@ #include #include "MetaDaemonInit.h" +#include "clients/meta/MetaClient.h" #include "common/base/Base.h" #include "common/base/SignalHandler.h" #include "common/fs/FileUtils.h" @@ -25,7 +26,6 @@ #include "meta/KVBasedClusterIdMan.h" #include "meta/MetaServiceHandler.h" #include "meta/MetaVersionMan.h" -#include "meta/RootUserMan.h" #include "meta/http/MetaHttpReplaceHostHandler.h" #include "meta/processors/job/JobManager.h" #include "meta/stats/MetaStats.h" @@ -167,29 +167,10 @@ int main(int argc, char* argv[]) { } } - { - /** - * Only leader part needed. - */ - auto ret = gKVStore->partLeader(nebula::kDefaultSpaceId, nebula::kDefaultPartId); - if (!nebula::ok(ret)) { - LOG(ERROR) << "Part leader get failed"; - return EXIT_FAILURE; - } - if (nebula::value(ret) == localhost) { - LOG(INFO) << "Check and init root user"; - auto checkRet = nebula::meta::RootUserMan::isGodExists(gKVStore.get()); - if (!nebula::ok(checkRet)) { - auto retCode = nebula::error(checkRet); - LOG(ERROR) << "Parser God Role error:" << apache::thrift::util::enumNameSafe(retCode); - return EXIT_FAILURE; - } - auto existGod = nebula::value(checkRet); - if (!existGod && !nebula::meta::RootUserMan::initRootUser(gKVStore.get())) { - LOG(ERROR) << "Init root user failed"; - return EXIT_FAILURE; - } - } + auto godInit = initGodUser(gKVStore.get(), localhost); + if (godInit != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Init god user failed"; + return EXIT_FAILURE; } auto metaServer = std::make_unique(); diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index ef587ec6bda..fe76db5f26b 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -10,6 +10,7 @@ #include "common/base/Base.h" #include "common/base/SignalHandler.h" +#include "common/datatypes/HostAddr.h" #include "common/fs/FileUtils.h" #include "common/hdfs/HdfsCommandHelper.h" #include "common/hdfs/HdfsHelper.h" @@ -17,12 +18,14 @@ #include "common/ssl/SSLConfig.h" #include "common/thread/GenericThreadPool.h" #include "common/utils/MetaKeyUtils.h" +#include "kvstore/KVStore.h" #include "kvstore/NebulaStore.h" #include "kvstore/PartManager.h" #include "meta/ActiveHostsMan.h" #include "meta/KVBasedClusterIdMan.h" #include "meta/MetaServiceHandler.h" #include "meta/MetaVersionMan.h" +#include "meta/RootUserMan.h" #include "meta/http/MetaHttpReplaceHostHandler.h" #include "meta/processors/job/JobManager.h" #include "meta/stats/MetaStats.h" @@ -46,6 +49,8 @@ DECLARE_string(meta_server_addrs); // use define from grap flags. DEFINE_int32(ws_meta_http_port, 11000, "Port to listen on Meta with HTTP protocol"); #endif +DECLARE_uint32(raft_heartbeat_interval_secs); + using nebula::web::PathParams; namespace nebula::meta { @@ -158,6 +163,57 @@ std::unique_ptr initKV(std::vector p return kvstore; } +nebula::cpp2::ErrorCode initGodUser(nebula::kvstore::KVStore* kvstore, + const nebula::HostAddr& localhost) { + const int kMaxRetryTime = FLAGS_raft_heartbeat_interval_secs * 3; + constexpr int kRetryInterval = 1; + int retryTime = 0; + // Both leader & follower need to wait all reading ok. + // leader init need to retry writing if leader changed. + while (true) { + retryTime += kRetryInterval; + if (retryTime > kMaxRetryTime) { + LOG(ERROR) << "Retry too many times"; + return nebula::cpp2::ErrorCode::E_RETRY_EXHAUSTED; + } + auto ret = kvstore->partLeader(nebula::kDefaultSpaceId, nebula::kDefaultPartId); + if (!nebula::ok(ret)) { + LOG(ERROR) << "Part leader get failed"; + return nebula::error(ret); + } + LOG(INFO) << "Check root user"; // follower need to wait reading all ok, too. + auto checkRet = nebula::meta::RootUserMan::isGodExists(kvstore); + if (!nebula::ok(checkRet)) { + auto retCode = nebula::error(checkRet); + if (retCode == nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + LOG(INFO) << "Leader changed, retry"; + sleep(kRetryInterval); + continue; + } + LOG(ERROR) << "Parser God Role error:" << apache::thrift::util::enumNameSafe(retCode); + return nebula::error(checkRet); + } + if (nebula::value(ret) == localhost) { + auto existGod = nebula::value(checkRet); + if (!existGod) { + auto initGod = nebula::meta::RootUserMan::initRootUser(kvstore); + if (initGod != nebula::cpp2::ErrorCode::SUCCEEDED) { + if (initGod != nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { + LOG(ERROR) << "Init God Role error:" << apache::thrift::util::enumNameSafe(initGod); + return initGod; + } else { + LOG(INFO) << "Leader changed, retry"; + sleep(kRetryInterval); + continue; + } + } + } + } + break; + } + return nebula::cpp2::ErrorCode::SUCCEEDED; +} + nebula::Status initWebService(nebula::WebService* svc, nebula::kvstore::KVStore* kvstore) { LOG(INFO) << "Starting Meta HTTP Service"; auto& router = svc->router(); diff --git a/src/daemons/MetaDaemonInit.h b/src/daemons/MetaDaemonInit.h index 0a94ae4bbd4..dfcd8077e06 100644 --- a/src/daemons/MetaDaemonInit.h +++ b/src/daemons/MetaDaemonInit.h @@ -9,6 +9,7 @@ #include "common/base/Status.h" #include "common/hdfs/HdfsCommandHelper.h" +#include "interface/gen-cpp2/common_types.h" #include "kvstore/KVStore.h" #include "webservice/WebService.h" @@ -18,4 +19,7 @@ std::unique_ptr initKV(std::vector p nebula::HostAddr localhost); nebula::Status initWebService(nebula::WebService* svc, nebula::kvstore::KVStore* kvstore); + +nebula::cpp2::ErrorCode initGodUser(nebula::kvstore::KVStore* kvstore, + const nebula::HostAddr& localhost); #endif diff --git a/src/daemons/StandAloneDaemon.cpp b/src/daemons/StandAloneDaemon.cpp index 5b519f726f5..f85b9f4fddc 100644 --- a/src/daemons/StandAloneDaemon.cpp +++ b/src/daemons/StandAloneDaemon.cpp @@ -223,29 +223,10 @@ int main(int argc, char *argv[]) { } } - { - /** - * Only leader part needed. - */ - auto ret = gMetaKVStore->partLeader(nebula::kDefaultSpaceId, nebula::kDefaultPartId); - if (!nebula::ok(ret)) { - LOG(ERROR) << "Part leader get failed"; - return; - } - if (nebula::value(ret) == metaLocalhost) { - LOG(INFO) << "Check and init root user"; - auto checkRet = nebula::meta::RootUserMan::isGodExists(gMetaKVStore.get()); - if (!nebula::ok(checkRet)) { - auto retCode = nebula::error(checkRet); - LOG(ERROR) << "Parser God Role error:" << apache::thrift::util::enumNameSafe(retCode); - return; - } - auto existGod = nebula::value(checkRet); - if (!existGod && !nebula::meta::RootUserMan::initRootUser(gMetaKVStore.get())) { - LOG(ERROR) << "Init root user failed"; - return; - } - } + auto godInit = initGodUser(gMetaKVStore.get(), metaLocalhost); + if (godInit != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Init god user failed"; + return; } auto handler = diff --git a/src/graph/planner/ngql/LookupPlanner.cpp b/src/graph/planner/ngql/LookupPlanner.cpp index 6bba68baed2..ee4282ef704 100644 --- a/src/graph/planner/ngql/LookupPlanner.cpp +++ b/src/graph/planner/ngql/LookupPlanner.cpp @@ -58,8 +58,11 @@ StatusOr LookupPlanner::transform(AstContext* astCtx) { if (lookupCtx->filter) { plan.root = Filter::make(qctx, plan.root, lookupCtx->filter); } - plan.root = Project::make(qctx, plan.root, lookupCtx->yieldExpr); + if (lookupCtx->dedup) { + plan.root = Dedup::make(qctx, plan.root); + } + return plan; } diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp index 5ea5fbc1ad3..eecddc5bbd9 100644 --- a/src/graph/service/GraphFlags.cpp +++ b/src/graph/service/GraphFlags.cpp @@ -103,3 +103,5 @@ DEFINE_uint32( gc_worker_size, 0, "Background garbage clean workers, default number is 0 which means using hardware core size."); + +DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key"); diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h index f2c7ecf9c75..07e7bbdb3ff 100644 --- a/src/graph/service/GraphFlags.h +++ b/src/graph/service/GraphFlags.h @@ -65,4 +65,7 @@ DECLARE_int32(max_job_size); DECLARE_bool(enable_async_gc); DECLARE_uint32(gc_worker_size); + +DECLARE_bool(graph_use_vertex_key); + #endif // GRAPH_GRAPHFLAGS_H_ diff --git a/src/graph/validator/LookupValidator.cpp b/src/graph/validator/LookupValidator.cpp index 9a5a73fd93d..a3a9500cb97 100644 --- a/src/graph/validator/LookupValidator.cpp +++ b/src/graph/validator/LookupValidator.cpp @@ -442,8 +442,7 @@ StatusOr LookupValidator::checkConstExpr(Expression* expr, if (type == nebula::cpp2::PropertyType::UNKNOWN) { return Status::SemanticError("Invalid column: %s", prop.c_str()); } - QueryExpressionContext dummy(nullptr); - auto v = Expression::eval(expr, dummy); + auto v = Expression::eval(expr, QueryExpressionContext(qctx_->ectx())()); // TODO(Aiee) extract the type cast logic as a method if we decide to support // more cross-type comparisons. diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp index 41f1f6fb14f..d59072d83a9 100644 --- a/src/graph/validator/MutateValidator.cpp +++ b/src/graph/validator/MutateValidator.cpp @@ -46,6 +46,9 @@ Status InsertVerticesValidator::check() { } auto tagItems = sentence->tagItems(); + if (!FLAGS_graph_use_vertex_key && tagItems.empty()) { + return Status::SemanticError("Insert vertex is forbidden, please speicify the tag"); + } schemas_.reserve(tagItems.size()); diff --git a/src/graph/visitor/PrunePropertiesVisitor.cpp b/src/graph/visitor/PrunePropertiesVisitor.cpp index 5970afb138c..4a1f0df53d6 100644 --- a/src/graph/visitor/PrunePropertiesVisitor.cpp +++ b/src/graph/visitor/PrunePropertiesVisitor.cpp @@ -358,11 +358,13 @@ void PrunePropertiesVisitor::pruneCurrent(AppendVertices *node) { node->setVertexProps(nullptr); } else { // only get _tag when props is nullptr - auto tagId = vertexProps->front().tag_ref().value(); - VertexProp newVProp; - newVProp.tag_ref() = tagId; - newVProp.props_ref() = {nebula::kTag}; - prunedVertexProps->emplace_back(std::move(newVProp)); + for (auto &vertexProp : *vertexProps) { + auto tagId = vertexProp.tag_ref().value(); + VertexProp newVProp; + newVProp.tag_ref() = tagId; + newVProp.props_ref() = {nebula::kTag}; + prunedVertexProps->emplace_back(std::move(newVProp)); + } node->setVertexProps(std::move(prunedVertexProps)); } return; @@ -373,11 +375,13 @@ void PrunePropertiesVisitor::pruneCurrent(AppendVertices *node) { node->setVertexProps(nullptr); } else { // only get _tag when props is nullptr - auto tagId = vertexProps->front().tag_ref().value(); - VertexProp newVProp; - newVProp.tag_ref() = tagId; - newVProp.props_ref() = {nebula::kTag}; - prunedVertexProps->emplace_back(std::move(newVProp)); + for (auto &vertexProp : *vertexProps) { + auto tagId = vertexProp.tag_ref().value(); + VertexProp newVProp; + newVProp.tag_ref() = tagId; + newVProp.props_ref() = {nebula::kTag}; + prunedVertexProps->emplace_back(std::move(newVProp)); + } node->setVertexProps(std::move(prunedVertexProps)); } return; diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index a1449b57e2f..0015df99be7 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -1988,7 +1988,7 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, lastTotalSize_ = 0; } else if (lastSnapshotCommitId_ != req.get_committed_log_id() || lastSnapshotCommitTerm_ != req.get_committed_log_term()) { - // Still waiting for snapshot from another peer, just return error. If the the peer doesn't + // Still waiting for snapshot from another peer, just return error. If the peer doesn't // send any logs during raft_snapshot_timeout, will convert to Status::RUNNING, so we can accept // snapshot again resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; diff --git a/src/meta/RootUserMan.h b/src/meta/RootUserMan.h index c8bde933203..af3e454f312 100644 --- a/src/meta/RootUserMan.h +++ b/src/meta/RootUserMan.h @@ -10,6 +10,7 @@ #include "common/base/Base.h" #include "common/utils/MetaKeyUtils.h" +#include "interface/gen-cpp2/common_types.h" #include "kvstore/KVStore.h" namespace nebula { @@ -53,7 +54,7 @@ class RootUserMan { } } - static bool initRootUser(kvstore::KVStore* kv) { + static nebula::cpp2::ErrorCode initRootUser(kvstore::KVStore* kv) { LOG(INFO) << "Init root user"; auto encodedPwd = proxygen::md5Encode(folly::StringPiece("nebula")); auto userKey = MetaKeyUtils::userKey("root"); @@ -64,18 +65,15 @@ class RootUserMan { std::vector data; data.emplace_back(std::move(userKey), std::move(userVal)); data.emplace_back(std::move(roleKey), std::move(roleVal)); - bool ret = true; + nebula::cpp2::ErrorCode ec; folly::Baton baton; kv->asyncMultiPut( kDefaultSpaceId, kDefaultPartId, std::move(data), [&](nebula::cpp2::ErrorCode code) { - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Put failed, error " << static_cast(code); - ret = false; - } + ec = code; baton.post(); }); baton.wait(); - return ret; + return ec; } }; diff --git a/src/storage/CompactionFilter.h b/src/storage/CompactionFilter.h index 585077e36fe..7fc9cec17e4 100644 --- a/src/storage/CompactionFilter.h +++ b/src/storage/CompactionFilter.h @@ -14,6 +14,7 @@ #include "common/utils/OperationKeyUtils.h" #include "kvstore/CompactionFilter.h" #include "storage/CommonUtils.h" +#include "storage/StorageFlags.h" namespace nebula { namespace storage { @@ -36,6 +37,8 @@ class StorageCompactionFilter final : public kvstore::KVFilter { return !edgeValid(spaceId, key, val); } else if (IndexKeyUtils::isIndexKey(key)) { return !indexValid(spaceId, key, val); + } else if (!FLAGS_use_vertex_key && NebulaKeyUtils::isVertex(key)) { + return true; } else if (NebulaKeyUtils::isLock(vIdLen_, key)) { return !lockValid(spaceId, key); } else { diff --git a/src/storage/StorageFlags.cpp b/src/storage/StorageFlags.cpp index 2aa3a35f4bc..bd100191c9f 100644 --- a/src/storage/StorageFlags.cpp +++ b/src/storage/StorageFlags.cpp @@ -46,3 +46,5 @@ DEFINE_bool(query_concurrently, false, "whether to run query of each part concurrently, only lookup and " "go are supported"); + +DEFINE_bool(use_vertex_key, false, "whether allow insert or query the vertex key"); diff --git a/src/storage/StorageFlags.h b/src/storage/StorageFlags.h index 7cbb1bb0137..197f9318c5c 100644 --- a/src/storage/StorageFlags.h +++ b/src/storage/StorageFlags.h @@ -34,4 +34,6 @@ DECLARE_int32(max_edge_returned_per_vertex); DECLARE_bool(query_concurrently); +DECLARE_bool(use_vertex_key); + #endif // STORAGE_STORAGEFLAGS_H_ diff --git a/src/storage/admin/StatsTask.cpp b/src/storage/admin/StatsTask.cpp index 9b9bb304394..c0cf60789de 100644 --- a/src/storage/admin/StatsTask.cpp +++ b/src/storage/admin/StatsTask.cpp @@ -111,8 +111,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId, std::unique_ptr tagIter; auto edgePrefix = NebulaKeyUtils::edgePrefix(part); std::unique_ptr edgeIter; - auto vertexPrefix = NebulaKeyUtils::vertexPrefix(part); - std::unique_ptr vertexIter; // When the storage occurs leader change, continue to read data from the // follower instead of reporting an error. @@ -126,11 +124,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId, LOG(INFO) << "Stats task failed"; return ret; } - ret = env_->kvstore_->prefix(spaceId, part, vertexPrefix, &vertexIter, true); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Stats task failed"; - return ret; - } std::unordered_map tagsVertices; std::unordered_map edgetypeEdges; std::unordered_map positiveRelevancy; @@ -173,6 +166,7 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId, tagIter->next(); continue; } + spaceVertices++; tagsVertices[tagId] += 1; tagIter->next(); sleepIfScannedSomeRecord(++countToSleep); @@ -227,11 +221,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId, edgeIter->next(); sleepIfScannedSomeRecord(++countToSleep); } - while (vertexIter && vertexIter->valid()) { - spaceVertices++; - vertexIter->next(); - sleepIfScannedSomeRecord(++countToSleep); - } nebula::meta::cpp2::StatsItem statsItem; // convert tagId/edgeType to tagName/edgeName diff --git a/src/storage/exec/GetPropNode.h b/src/storage/exec/GetPropNode.h index a8b226f2dcc..d5560f8f361 100644 --- a/src/storage/exec/GetPropNode.h +++ b/src/storage/exec/GetPropNode.h @@ -42,18 +42,23 @@ class GetTagPropNode : public QueryNode { return ret; } - // if none of the tag node and vertex valid, do not emplace the row + // If none of the tag node valid, will check vertex key if use_vertex_key is true, + // do not emplace the row if the flag is false if (!std::any_of(tagNodes_.begin(), tagNodes_.end(), [](const auto& tagNode) { return tagNode->valid(); })) { - auto kvstore = context_->env()->kvstore_; - auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId); - std::string value; - ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value); - if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + if (FLAGS_use_vertex_key) { + auto kvstore = context_->env()->kvstore_; + auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId); + std::string value; + ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value); + if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } + } else { return nebula::cpp2::ErrorCode::SUCCEEDED; - } else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - return ret; } } diff --git a/src/storage/exec/HashJoinNode.h b/src/storage/exec/HashJoinNode.h index 32f8a36224a..824f5e4a867 100644 --- a/src/storage/exec/HashJoinNode.h +++ b/src/storage/exec/HashJoinNode.h @@ -19,7 +19,7 @@ namespace storage { // several SingleEdgeNode of different edge types all edges of a vertex. The // output would be the result of tag, it is a List, each cell save a list of // property values, if tag not found, it will be a empty value. Also it will -// return a iterator of edges which can pass ttl check and ready to be read. +// return an iterator of edges which can pass ttl check and ready to be read. class HashJoinNode : public IterateNode { public: using RelNode::doExecute; diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h index 876d788952b..0c643803729 100644 --- a/src/storage/exec/UpdateNode.h +++ b/src/storage/exec/UpdateNode.h @@ -488,7 +488,6 @@ class UpdateTagNode : public UpdateNode { } } // step 3, insert new vertex data - batchHolder->put(NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId), ""); batchHolder->put(std::move(key_), std::move(nVal)); return encodeBatchValue(batchHolder->getBatch()); } diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp index 3f598fcad12..354d5f1a6f3 100644 --- a/src/storage/mutate/AddVerticesProcessor.cpp +++ b/src/storage/mutate/AddVerticesProcessor.cpp @@ -81,7 +81,6 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) { code = nebula::cpp2::ErrorCode::E_INVALID_VID; break; } - data.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid), ""); for (auto& newTag : newTags) { auto tagId = newTag.get_tag_id(); VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId; @@ -160,7 +159,6 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re break; } - verticeData.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid)); for (const auto& newTag : newTags) { auto tagId = newTag.get_tag_id(); VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId; diff --git a/src/storage/test/IndexWriteTest.cpp b/src/storage/test/IndexWriteTest.cpp index 486fda4702a..0b3301b1c21 100644 --- a/src/storage/test/IndexWriteTest.cpp +++ b/src/storage/test/IndexWriteTest.cpp @@ -244,7 +244,7 @@ TEST(IndexTest, VerticesValueTest) { auto* schemaMan = reinterpret_cast(env->schemaMan_); schemaMan->addTagSchema(spaceId, tagId, mock::MockData::mockTypicaSchemaV2(pool)); } - // Mock a index for nullable column and default column. + // Mock an index for nullable column and default column. { auto* indexMan = reinterpret_cast(env->indexMan_); indexMan->addTagIndex(spaceId, tagId, indexId, mock::MockData::mockTypicaIndexColumns()); @@ -363,7 +363,7 @@ TEST(IndexTest, AlterTagIndexTest) { auto* schemaMan = reinterpret_cast(env->schemaMan_); schemaMan->addTagSchema(spaceId, tagId, mock::MockData::mockGeneralTagSchemaV1()); } - // Mock a index for nullable column and default column. + // Mock an index for nullable column and default column. { auto* indexMan = reinterpret_cast(env->indexMan_); indexMan->addTagIndex(spaceId, tagId, indexId1, mock::MockData::mockGeneralTagIndexColumns()); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 27d5728574a..84b3def1d70 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -65,4 +65,3 @@ nebula_add_subdirectory(simple-kv-verify) endif() nebula_add_subdirectory(meta-dump) nebula_add_subdirectory(db-dump) -nebula_add_subdirectory(db-upgrade) diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt deleted file mode 100644 index af0b1efbd26..00000000000 --- a/src/tools/db-upgrade/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -nebula_add_executable( - NAME - db_upgrader - SOURCES - DbUpgraderTool.cpp - NebulaKeyUtilsV1.cpp - NebulaKeyUtilsV2.cpp - NebulaKeyUtilsV3.cpp - DbUpgrader.cpp - OBJECTS - ${tools_test_deps} - LIBRARIES - ${ROCKSDB_LIBRARIES} - ${THRIFT_LIBRARIES} - ${PROXYGEN_LIBRARIES} -) - -install( - TARGETS - db_upgrader - PERMISSIONS - OWNER_EXECUTE OWNER_WRITE OWNER_READ - GROUP_EXECUTE GROUP_READ - WORLD_EXECUTE WORLD_READ - DESTINATION - bin - COMPONENT - tool -) diff --git a/src/tools/db-upgrade/DbUpgrader.cpp b/src/tools/db-upgrade/DbUpgrader.cpp deleted file mode 100644 index c0834d87339..00000000000 --- a/src/tools/db-upgrade/DbUpgrader.cpp +++ /dev/null @@ -1,1253 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "tools/db-upgrade/DbUpgrader.h" - -#include "common/datatypes/Value.h" -#include "common/fs/FileUtils.h" -#include "common/utils/IndexKeyUtils.h" -#include "common/utils/NebulaKeyUtils.h" -#include "rocksdb/sst_file_writer.h" -#include "tools/db-upgrade/NebulaKeyUtilsV1.h" -#include "tools/db-upgrade/NebulaKeyUtilsV2.h" -#include "tools/db-upgrade/NebulaKeyUtilsV3.h" - -DEFINE_string(src_db_path, - "", - "Source data path(data_path in storage 1.x conf), " - "multi paths should be split by comma"); -DEFINE_string(dst_db_path, - "", - "Destination data path(data_path in storage 2.0 conf), " - "multi paths should be split by comma"); -DEFINE_string(upgrade_meta_server, "127.0.0.1:45500", "Meta servers' address."); -DEFINE_uint32(write_batch_num, 100, "The size of the batch written to rocksdb"); -DEFINE_string(upgrade_version, - "", - "When the value is 1:2, upgrade the data from 1.x to 2.0 GA. " - "When the value is 2RC:2, upgrade the data from 2.0 RC to 2.0 GA." - "When the value is 2:3, upgrade the data from 2.0 GA to 3.0 ."); -DEFINE_bool(compactions, - true, - "When the upgrade of the space is completed, " - "whether to compact data"); -DEFINE_uint32(max_concurrent_parts, 10, "The parts could be processed simultaneously"); -DEFINE_uint32(max_concurrent_spaces, 5, "The spaces could be processed simultaneously"); - -namespace nebula { -namespace storage { - -using nebula::cpp2::PropertyType; - -Status UpgraderSpace::init(meta::MetaClient* mclient, - meta::ServerBasedSchemaManager* sMan, - meta::IndexManager* iMan, - const std::string& srcPath, - const std::string& dstPath, - const std::string& entry) { - metaClient_ = mclient; - schemaMan_ = sMan; - indexMan_ = iMan; - srcPath_ = srcPath; - dstPath_ = dstPath; - entry_ = entry; - - auto ret = initSpace(entry_); - if (!ret.ok()) { - LOG(ERROR) << "Init " << srcPath << " space id " << entry_ << " failed"; - return ret; - } - - return Status::OK(); -} - -Status UpgraderSpace::initSpace(const std::string& sId) { - try { - spaceId_ = folly::to(sId); - } catch (const std::exception& ex) { - LOG(ERROR) << "Cannot convert space id " << sId; - return Status::Error("Cannot convert space id %s", sId.c_str()); - } - - auto sRet = schemaMan_->toGraphSpaceName(spaceId_); - if (!sRet.ok()) { - LOG(ERROR) << "Space id " << spaceId_ << " no found"; - return sRet.status(); - } - spaceName_ = sRet.value(); - - auto spaceVidLen = metaClient_->getSpaceVidLen(spaceId_); - if (!spaceVidLen.ok()) { - return spaceVidLen.status(); - } - spaceVidLen_ = spaceVidLen.value(); - - // Use readonly rocksdb - readEngine_.reset(new nebula::kvstore::RocksEngine( - spaceId_, spaceVidLen_, srcPath_, "", nullptr, nullptr, false)); - writeEngine_.reset(new nebula::kvstore::RocksEngine(spaceId_, spaceVidLen_, dstPath_)); - - parts_.clear(); - parts_ = readEngine_->allParts(); - LOG(INFO) << "Src data path: " << srcPath_ << " space id " << spaceId_ << " has " << parts_.size() - << " parts"; - - tagSchemas_.clear(); - tagFieldName_.clear(); - tagIndexes_.clear(); - edgeSchemas_.clear(); - edgeFieldName_.clear(); - edgeIndexes_.clear(); - - auto ret = buildSchemaAndIndex(); - if (!ret.ok()) { - LOG(ERROR) << "Build schema and index in space id " << spaceId_ << " failed"; - return ret; - } - - pool_ = std::make_unique(FLAGS_max_concurrent_parts); - // Parallel process part - for (auto& partId : parts_) { - partQueue_.add(partId); - } - - return Status::OK(); -} - -Status UpgraderSpace::buildSchemaAndIndex() { - // Get all tag in space - auto tags = schemaMan_->getAllVerTagSchema(spaceId_); - if (!tags.ok()) { - LOG(ERROR) << "Space id " << spaceId_ << " no found"; - return tags.status(); - } - tagSchemas_ = std::move(tags).value(); - - for (auto& tag : tagSchemas_) { - auto tagId = tag.first; - auto newestTagschema = tag.second.back(); - auto fields = newestTagschema->getNumFields(); - for (size_t i = 0; i < fields; i++) { - tagFieldName_[tagId].emplace_back(newestTagschema->getFieldName(i)); - } - if (fields == 0) { - tagFieldName_[tagId] = {}; - } - LOG(INFO) << "Tag id " << tagId << " has " << tagFieldName_[tagId].size() << " fields!"; - } - - // Get all tag index in space - std::vector> tagIndexes; - auto iRet = indexMan_->getTagIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << "Space id " << spaceId_ << " no found"; - return iRet.status(); - } - tagIndexes = std::move(iRet).value(); - - // Handle tag index - for (auto& tagIndex : tagIndexes) { - auto tagId = tagIndex->get_schema_id().get_tag_id(); - tagIndexes_[tagId].emplace(tagIndex); - } - - for (auto& tagindexes : tagIndexes_) { - LOG(INFO) << "Tag id " << tagindexes.first << " has " << tagindexes.second.size() << " indexes"; - } - - // Get all edge in space - auto edges = schemaMan_->getAllVerEdgeSchema(spaceId_); - if (!edges.ok()) { - LOG(ERROR) << "Space id " << spaceId_ << " no found"; - return edges.status(); - } - edgeSchemas_ = std::move(edges).value(); - - for (auto& edge : edgeSchemas_) { - auto edgetype = edge.first; - auto newestEdgeSchema = edge.second.back(); - auto fields = newestEdgeSchema->getNumFields(); - for (size_t i = 0; i < fields; i++) { - edgeFieldName_[edgetype].emplace_back(newestEdgeSchema->getFieldName(i)); - } - if (fields == 0) { - edgeFieldName_[edgetype] = {}; - } - LOG(INFO) << "Edgetype " << edgetype << " has " << edgeFieldName_[edgetype].size() - << " fields!"; - } - - // Get all edge index in space - std::vector> edgeIndexes; - iRet = indexMan_->getEdgeIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << "Space id " << spaceId_ << " no found"; - return iRet.status(); - } - edgeIndexes = std::move(iRet).value(); - - // Handle edge index - for (auto& edgeIndex : edgeIndexes) { - auto edgetype = edgeIndex->get_schema_id().get_edge_type(); - edgeIndexes_[edgetype].emplace(edgeIndex); - } - - for (auto& edgeindexes : edgeIndexes_) { - LOG(INFO) << "EdgeType " << edgeindexes.first << " has " << edgeindexes.second.size() - << " indexes"; - } - return Status::OK(); -} - -bool UpgraderSpace::isValidVidLen(VertexID srcVId, VertexID dstVId) { - if (!NebulaKeyUtils::isValidVidLen(spaceVidLen_, srcVId, dstVId)) { - LOG(ERROR) << "Vertex id length is illegal, expect: " << spaceVidLen_ << " result: " << srcVId - << " " << dstVId; - return false; - } - return true; -} - -void UpgraderSpace::runPartV1() { - std::chrono::milliseconds take_dura{10}; - if (auto pId = partQueue_.try_take_for(take_dura)) { - PartitionID partId = *pId; - // Handle vertex and edge, if there is an index, generate index data - LOG(INFO) << "Start to handle vertex/edge/index data in space id " << spaceId_ << " part id " - << partId; - const auto& prefix = NebulaKeyUtilsV1::prefix(partId); - std::unique_ptr iter; - auto retCode = readEngine_->prefix(prefix, &iter); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Space id " << spaceId_ << " part " << partId << " no found!"; - LOG(ERROR) << "Handle vertex/edge/index data in space id " << spaceId_ << " part id " - << partId << " failed"; - - auto unFinishedPart = --unFinishedPart_; - if (unFinishedPart == 0) { - // all parts has finished - LOG(INFO) << "Handle last part: " << partId << " vertex/edge/index data in space id " - << spaceId_ << " finished"; - } else { - pool_->add(std::bind(&UpgraderSpace::runPartV1, this)); - } - return; - } - - std::vector data; - TagID lastTagId = 0; - int64_t lastVertexId = 0; - - int64_t lastSrcVertexId = 0; - EdgeType lastEdgeType = 0; - int64_t lastDstVertexId = 0; - EdgeRanking lastRank = 0; - - while (iter && iter->valid()) { - auto key = iter->key(); - if (NebulaKeyUtilsV1::isVertex(key)) { - auto vId = NebulaKeyUtilsV1::getVertexId(key); - auto tagId = NebulaKeyUtilsV1::getTagId(key); - - auto it = tagSchemas_.find(tagId); - if (it == tagSchemas_.end()) { - // Invalid data - iter->next(); - continue; - } - auto iterField = tagFieldName_.find(tagId); - if (iterField == tagFieldName_.end()) { - // Invalid data - iter->next(); - continue; - } - if (vId == lastVertexId && tagId == lastTagId) { - // Multi version - iter->next(); - continue; - } - - auto strVid = std::string(reinterpret_cast(&vId), sizeof(vId)); - auto newTagSchema = it->second.back().get(); - // Generate 2.0 key - auto newKey = NebulaKeyUtils::tagKey(spaceVidLen_, partId, strVid, tagId); - auto val = iter->val(); - auto reader = RowReaderWrapper::getTagPropReader(schemaMan_, spaceId_, tagId, val); - if (!reader) { - LOG(ERROR) << "Can't get tag reader of " << tagId; - iter->next(); - continue; - } - // Generate 2.0 value and index records - encodeVertexValue(partId, reader.get(), newTagSchema, newKey, strVid, tagId, data); - - lastTagId = tagId; - lastVertexId = vId; - } else if (NebulaKeyUtilsV1::isEdge(key)) { - auto svId = NebulaKeyUtilsV1::getSrcId(key); - auto edgetype = NebulaKeyUtilsV1::getEdgeType(key); - auto ranking = NebulaKeyUtilsV1::getRank(key); - auto dvId = NebulaKeyUtilsV1::getDstId(key); - - auto it = edgeSchemas_.find(std::abs(edgetype)); - if (it == edgeSchemas_.end()) { - // Invalid data - iter->next(); - continue; - } - auto iterField = edgeFieldName_.find(std::abs(edgetype)); - if (iterField == edgeFieldName_.end()) { - // Invalid data - iter->next(); - continue; - } - if (svId == lastSrcVertexId && edgetype == lastEdgeType && ranking == lastRank && - dvId == lastDstVertexId) { - // Multi version - iter->next(); - continue; - } - - auto strsvId = std::string(reinterpret_cast(&svId), sizeof(svId)); - auto strdvId = std::string(reinterpret_cast(&dvId), sizeof(dvId)); - - auto newEdgeSchema = it->second.back().get(); - - // Generate 2.0 key - auto newKey = - NebulaKeyUtils::edgeKey(spaceVidLen_, partId, strsvId, edgetype, ranking, strdvId); - auto val = iter->val(); - auto reader = - RowReaderWrapper::getEdgePropReader(schemaMan_, spaceId_, std::abs(edgetype), val); - if (!reader) { - LOG(ERROR) << "Can't get edge reader of " << edgetype; - iter->next(); - continue; - } - // Generate 2.0 value and index records - encodeEdgeValue( - partId, reader.get(), newEdgeSchema, newKey, strsvId, edgetype, ranking, strdvId, data); - lastSrcVertexId = svId; - lastEdgeType = edgetype; - lastRank = ranking; - lastDstVertexId = dvId; - } - - if (data.size() >= FLAGS_write_batch_num) { - VLOG(2) << "Send record total rows " << data.size(); - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " part id " << partId - << " failed."; - } - data.clear(); - } - - iter->next(); - } - - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " part id " << partId - << " failed."; - } - data.clear(); - LOG(INFO) << "Handle vertex/edge/index data in space id " << spaceId_ << " part id " << partId - << " finished"; - - auto unFinishedPart = --unFinishedPart_; - if (unFinishedPart == 0) { - // all parts has finished - LOG(INFO) << "Handle last part: " << partId << " vertex/edge/index data in space id " - << spaceId_ << " finished"; - } else { - pool_->add(std::bind(&UpgraderSpace::runPartV1, this)); - } - } else { - LOG(INFO) << "Handle vertex/edge/index of parts data in space id " << spaceId_ << " finished"; - } -} - -void UpgraderSpace::doProcessV1() { - LOG(INFO) << "Start to handle data in space id " << spaceId_; - - // Parallel process part - auto partConcurrency = std::min(static_cast(FLAGS_max_concurrent_parts), parts_.size()); - LOG(INFO) << "Max concurrent parts: " << partConcurrency; - - unFinishedPart_ = parts_.size(); - - LOG(INFO) << "Start to handle vertex/edge/index of parts data in space id " << spaceId_; - for (size_t i = 0; i < partConcurrency; ++i) { - pool_->add(std::bind(&UpgraderSpace::runPartV1, this)); - } - - while (unFinishedPart_ != 0) { - sleep(10); - } - - // handle system data - { - LOG(INFO) << "Start to handle system data in space id " << spaceId_; - auto prefix = NebulaKeyUtilsV1::systemPrefix(); - std::unique_ptr iter; - auto retCode = readEngine_->prefix(prefix, &iter); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Space id " << spaceId_ << " get system data failed"; - LOG(ERROR) << "Handle system data in space id " << spaceId_ << " failed"; - return; - } - std::vector data; - while (iter && iter->valid()) { - auto key = iter->key(); - auto val = iter->val(); - data.emplace_back(std::move(key), std::move(val)); - if (data.size() >= FLAGS_write_batch_num) { - VLOG(2) << "Send system data total rows " << data.size(); - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " failed."; - } - data.clear(); - } - iter->next(); - } - - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " failed."; - } - LOG(INFO) << "Handle system data in space id " << spaceId_ << " success"; - LOG(INFO) << "Handle data in space id " << spaceId_ << " success"; - } -} - -void UpgraderSpace::runPartV2() { - std::chrono::milliseconds take_dura{10}; - if (auto pId = partQueue_.try_take_for(take_dura)) { - PartitionID partId = *pId; - // Handle vertex and edge, if there is an index, generate index data - LOG(INFO) << "Start to handle vertex/edge/index data in space id " << spaceId_ << " part id " - << partId; - auto prefix = NebulaKeyUtilsV2::partPrefix(partId); - std::unique_ptr iter; - auto retCode = readEngine_->prefix(prefix, &iter); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Space id " << spaceId_ << " part " << partId << " no found!"; - LOG(ERROR) << "Handle vertex/edge/index data in space id " << spaceId_ << " part id " - << partId << " failed"; - - auto unFinishedPart = --unFinishedPart_; - if (unFinishedPart == 0) { - // all parts has finished - LOG(INFO) << "Handle last part: " << partId << " vertex/edge/index data in space id " - << spaceId_ << " finished"; - } else { - pool_->add(std::bind(&UpgraderSpace::runPartV2, this)); - } - return; - } - - std::vector data; - TagID lastTagId = 0; - VertexID lastVertexId = ""; - - VertexID lastSrcVertexId = ""; - EdgeType lastEdgeType = 0; - VertexID lastDstVertexId = ""; - EdgeRanking lastRank = 0; - - while (iter && iter->valid()) { - auto key = iter->key(); - if (NebulaKeyUtilsV2::isVertex(spaceVidLen_, key)) { - auto vId = NebulaKeyUtilsV2::getVertexId(spaceVidLen_, key).str(); - auto tagId = NebulaKeyUtilsV2::getTagId(spaceVidLen_, key); - - auto it = tagSchemas_.find(tagId); - if (it == tagSchemas_.end()) { - // Invalid data - iter->next(); - continue; - } - auto iterField = tagFieldName_.find(tagId); - if (iterField == tagFieldName_.end()) { - // Invalid data - iter->next(); - continue; - } - if (vId == lastVertexId && tagId == lastTagId) { - // Multi version - iter->next(); - continue; - } - - auto newTagSchema = it->second.back().get(); - // Generate 2.0 key - auto newKey = NebulaKeyUtils::tagKey(spaceVidLen_, partId, vId, tagId); - auto val = iter->val(); - auto reader = RowReaderWrapper::getTagPropReader(schemaMan_, spaceId_, tagId, val); - if (!reader) { - LOG(ERROR) << "Can't get tag reader of " << tagId; - iter->next(); - continue; - } - // Generate 2.0 value and index records - encodeVertexValue(partId, reader.get(), newTagSchema, newKey, vId, tagId, data); - - lastTagId = tagId; - lastVertexId = vId; - } else if (NebulaKeyUtilsV2::isEdge(spaceVidLen_, key)) { - auto svId = NebulaKeyUtilsV2::getSrcId(spaceVidLen_, key).str(); - auto edgetype = NebulaKeyUtilsV2::getEdgeType(spaceVidLen_, key); - auto ranking = NebulaKeyUtilsV2::getRank(spaceVidLen_, key); - auto dvId = NebulaKeyUtilsV2::getDstId(spaceVidLen_, key).str(); - - auto it = edgeSchemas_.find(std::abs(edgetype)); - if (it == edgeSchemas_.end()) { - // Invalid data - iter->next(); - continue; - } - auto iterField = edgeFieldName_.find(std::abs(edgetype)); - if (iterField == edgeFieldName_.end()) { - // Invalid data - iter->next(); - continue; - } - if (svId == lastSrcVertexId && edgetype == lastEdgeType && ranking == lastRank && - dvId == lastDstVertexId) { - // Multi version - iter->next(); - continue; - } - - auto newEdgeSchema = it->second.back().get(); - - // Generate 2.0 key - auto newKey = NebulaKeyUtils::edgeKey(spaceVidLen_, partId, svId, edgetype, ranking, dvId); - auto val = iter->val(); - auto reader = - RowReaderWrapper::getEdgePropReader(schemaMan_, spaceId_, std::abs(edgetype), val); - if (!reader) { - LOG(ERROR) << "Can't get edge reader of " << edgetype; - iter->next(); - continue; - } - // Generate 2.0 value and index records - encodeEdgeValue( - partId, reader.get(), newEdgeSchema, newKey, svId, edgetype, ranking, dvId, data); - lastSrcVertexId = svId; - lastEdgeType = edgetype; - lastRank = ranking; - lastDstVertexId = dvId; - } - - if (data.size() >= FLAGS_write_batch_num) { - VLOG(2) << "Send record total rows " << data.size(); - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " part id " << partId - << " failed."; - } - data.clear(); - } - - iter->next(); - } - - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " part id " << partId - << " failed."; - } - data.clear(); - LOG(INFO) << "Handle vertex/edge/index data in space id " << spaceId_ << " part id " << partId - << " succeed"; - - auto unFinishedPart = --unFinishedPart_; - if (unFinishedPart == 0) { - // all parts has finished - LOG(INFO) << "Handle last part: " << partId << " vertex/edge/index data in space id " - << spaceId_ << " finished."; - } else { - pool_->add(std::bind(&UpgraderSpace::runPartV2, this)); - } - } else { - LOG(INFO) << "Handle vertex/edge/index of parts data in space id " << spaceId_ << " finished"; - } -} - -void UpgraderSpace::doProcessV2() { - LOG(INFO) << "Start to handle data in space id " << spaceId_; - - // Parallel process part - auto partConcurrency = std::min(static_cast(FLAGS_max_concurrent_parts), parts_.size()); - LOG(INFO) << "Max concurrent parts: " << partConcurrency; - unFinishedPart_ = parts_.size(); - - LOG(INFO) << "Start to handle vertex/edge/index of parts data in space id " << spaceId_; - for (size_t i = 0; i < partConcurrency; ++i) { - pool_->add(std::bind(&UpgraderSpace::runPartV2, this)); - } - - while (unFinishedPart_ != 0) { - sleep(10); - } - - // handle system data - { - LOG(INFO) << "Start to handle system data in space id " << spaceId_; - auto prefix = NebulaKeyUtilsV2::systemPrefix(); - std::unique_ptr iter; - auto retCode = readEngine_->prefix(prefix, &iter); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Space id " << spaceId_ << " get system data failed."; - LOG(ERROR) << "Handle system data in space id " << spaceId_ << " failed."; - return; - } - std::vector data; - while (iter && iter->valid()) { - auto key = iter->key(); - auto val = iter->val(); - data.emplace_back(std::move(key), std::move(val)); - if (data.size() >= FLAGS_write_batch_num) { - VLOG(2) << "Send system data total rows " << data.size(); - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " failed."; - } - data.clear(); - } - iter->next(); - } - - auto code = writeEngine_->multiPut(data); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Write multi put in space id " << spaceId_ << " failed."; - } - LOG(INFO) << "Handle system data in space id " << spaceId_ << " success"; - LOG(INFO) << "Handle data in space id " << spaceId_ << " success"; - } -} - -void UpgraderSpace::encodeVertexValue(PartitionID partId, - RowReader* reader, - const meta::NebulaSchemaProvider* schema, - std::string& newkey, - VertexID& strVid, - TagID tagId, - std::vector& data) { - // Get all returned field name - auto& fieldNames = tagFieldName_[tagId]; - - auto ret = encodeRowVal(reader, schema, fieldNames); - if (ret.empty()) { - LOG(ERROR) << "Vertex or edge value is empty"; - return; - } - data.emplace_back(std::move(newkey), ret); - - // encode v2 index value - auto it = tagIndexes_.find(tagId); - if (it != tagIndexes_.end()) { - // Use new RowReader - auto nReader = RowReaderWrapper::getTagPropReader(schemaMan_, spaceId_, tagId, ret); - if (nReader == nullptr) { - LOG(ERROR) << "Bad format row: space id " << spaceId_ << " tag id " << tagId << " vertex id " - << strVid; - return; - } - for (auto& index : it->second) { - auto newIndexKeys = indexVertexKeys(partId, strVid, nReader.get(), index, schema); - for (auto& newIndexKey : newIndexKeys) { - data.emplace_back(std::move(newIndexKey), ""); - } - } - } -} - -// If the field types are inconsistent, can be converted -WriteResult UpgraderSpace::convertValue(const meta::NebulaSchemaProvider* nSchema, - const meta::SchemaProviderIf* oSchema, - std::string& name, - Value& val) { - auto newpropType = nSchema->getFieldType(name); - auto oldpropType = oSchema->getFieldType(name); - if (newpropType == oldpropType) { - return WriteResult::SUCCEEDED; - } - - bool bval; - double fval; - int64_t ival; - std::string sval; - - // need convert - switch (val.type()) { - case Value::Type::NULLVALUE: - return WriteResult::SUCCEEDED; - case Value::Type::BOOL: { - switch (newpropType) { - case PropertyType::INT8: - case PropertyType::INT16: - case PropertyType::INT32: - case PropertyType::INT64: - case PropertyType::TIMESTAMP: - case PropertyType::VID: { - bval = val.getBool(); - if (bval) { - val.setInt(1); - } else { - val.setInt(0); - } - return WriteResult::SUCCEEDED; - } - case PropertyType::STRING: - case PropertyType::FIXED_STRING: { - try { - bval = val.getBool(); - sval = folly::to(bval); - val.setStr(sval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - case PropertyType::FLOAT: - case PropertyType::DOUBLE: { - try { - bval = val.getBool(); - fval = folly::to(bval); - val.setFloat(fval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - // other not need convert - default: - return WriteResult::SUCCEEDED; - } - } - case Value::Type::INT: { - switch (newpropType) { - case PropertyType::STRING: - case PropertyType::FIXED_STRING: { - try { - ival = val.getInt(); - sval = folly::to(ival); - val.setStr(sval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - // other not need convert - default: - return WriteResult::SUCCEEDED; - } - } - case Value::Type::FLOAT: { - switch (newpropType) { - case PropertyType::STRING: - case PropertyType::FIXED_STRING: { - try { - fval = val.getFloat(); - sval = folly::to(fval); - val.setStr(sval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - case PropertyType::BOOL: { - try { - fval = val.getFloat(); - bval = folly::to(fval); - val.setBool(bval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - // other not need convert - default: - return WriteResult::SUCCEEDED; - } - } - case Value::Type::STRING: { - switch (newpropType) { - case PropertyType::INT8: - case PropertyType::INT16: - case PropertyType::INT32: - case PropertyType::INT64: - case PropertyType::TIMESTAMP: - case PropertyType::VID: { - try { - sval = val.getStr(); - ival = folly::to(sval); - val.setInt(ival); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - case PropertyType::BOOL: { - try { - sval = val.getStr(); - bval = folly::to(sval); - val.setBool(bval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - case PropertyType::FLOAT: - case PropertyType::DOUBLE: { - try { - sval = val.getStr(); - fval = folly::to(sval); - val.setFloat(fval); - return WriteResult::SUCCEEDED; - } catch (const std::exception& e) { - return WriteResult::TYPE_MISMATCH; - } - } - // other not need convert - default: - return WriteResult::SUCCEEDED; - } - } - // other not need convert - default: - return WriteResult::SUCCEEDED; - } -} - -// Used for vertex and edge -std::string UpgraderSpace::encodeRowVal(const RowReader* reader, - const meta::NebulaSchemaProvider* schema, - std::vector& fieldName) { - auto oldSchema = reader->getSchema(); - if (oldSchema == nullptr) { - LOG(ERROR) << "Schema not found from RowReader."; - return ""; - } - - // encode v2 value, use new schema - WriteResult wRet; - RowWriterV2 rowWrite(schema); - - // fieldName contains all the fields of the latest schema. - // The data reader may not use the latest schema, - // If it does not contain the field, the default value or null value will be - // used. - for (auto& name : fieldName) { - auto val = reader->getValueByName(name); - if (val.type() != Value::Type::NULLVALUE) { - // If the field types are inconsistent, can be converted - wRet = convertValue(schema, oldSchema, name, val); - if (wRet != WriteResult::SUCCEEDED) { - LOG(ERROR) << "Convert value failed"; - return ""; - } - wRet = rowWrite.setValue(name, val); - if (wRet != WriteResult::SUCCEEDED) { - LOG(ERROR) << "Write rowWriterV2 failed"; - return ""; - } - } else { - // read null value - auto nullType = val.getNull(); - if (nullType == NullType::__NULL__) { - wRet = rowWrite.setValue(name, val); - if (wRet != WriteResult::SUCCEEDED) { - LOG(ERROR) << "Write rowWriterV2 failed"; - return ""; - } - } else if (nullType != NullType::UNKNOWN_PROP) { - // nullType == NullType::kNullUnknownProp, indicates that the field is - // only in the latest schema, maybe use default value or null value. - LOG(ERROR) << "Data is illegal in " << name << " field"; - return ""; - } - } - } - - wRet = rowWrite.finish(); - if (wRet != WriteResult::SUCCEEDED) { - LOG(ERROR) << "Write rowWriterV2 failed"; - return ""; - } - - return std::move(rowWrite).moveEncodedStr(); -} - -void UpgraderSpace::runPartV3() { - std::chrono::milliseconds take_dura{10}; - if (auto pId = partQueue_.try_take_for(take_dura)) { - PartitionID partId = *pId; - // Handle vertex and edge, if there is an index, generate index data - LOG(INFO) << "Start to handle vertex/edge/index data in space id " << spaceId_ << " part id " - << partId; - auto prefix = NebulaKeyUtilsV3::partTagPrefix(partId); - std::unique_ptr iter; - auto retCode = readEngine_->prefix(prefix, &iter); - if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Space id " << spaceId_ << " part " << partId << " no found!"; - LOG(ERROR) << "Handle vertex/edge/index data in space id " << spaceId_ << " part id " - << partId << " failed"; - - auto unFinishedPart = --unFinishedPart_; - if (unFinishedPart == 0) { - // all parts has finished - LOG(INFO) << "Handle last part: " << partId << " vertex/edge/index data in space id " - << spaceId_ << " finished"; - } else { - pool_->add(std::bind(&UpgraderSpace::runPartV3, this)); - } - return; - } - int64_t ingestFileCount = 0; - auto write_sst = [&, this](const std::vector& data) { - ::rocksdb::Options option; - option.create_if_missing = true; - option.compression = ::rocksdb::CompressionType::kNoCompression; - ::rocksdb::SstFileWriter sst_file_writer(::rocksdb::EnvOptions(), option); - std::string file = ::fmt::format(".nebula_upgrade.space-{}.part-{}-{}-{}.sst", - spaceId_, - partId, - ingestFileCount++, - std::time(nullptr)); - ::rocksdb::Status s = sst_file_writer.Open(file); - if (!s.ok()) { - LOG(FATAL) << "Faild upgrade V3 of space " << spaceId_ << ", part " << partId << ":" - << s.code(); - } - for (auto item : data) { - s = sst_file_writer.Put(item.first, item.second); - if (!s.ok()) { - LOG(FATAL) << "Faild upgrade V3 of space " << spaceId_ << ", part " << partId << ":" - << s.code(); - } - } - s = sst_file_writer.Finish(); - if (!s.ok()) { - LOG(FATAL) << "Faild upgrade V3 of space " << spaceId_ << ", part " << partId << ":" - << s.code(); - } - std::lock_guard lck(this->ingest_sst_file_mut_); - ingest_sst_file_.push_back(file); - }; - std::vector data; - std::string lastVertexKey = ""; - while (iter && iter->valid()) { - auto vertex = NebulaKeyUtilsV3::getVertexKey(iter->key()); - if (vertex == lastVertexKey) { - iter->next(); - continue; - } - data.emplace_back(vertex, ""); - lastVertexKey = vertex; - if (data.size() >= 100000) { - write_sst(data); - data.clear(); - } - iter->next(); - } - if (!data.empty()) { - write_sst(data); - data.clear(); - } - LOG(INFO) << "Handle vertex/edge/index data in space id " << spaceId_ << " part id " << partId - << " succeed"; - - auto unFinishedPart = --unFinishedPart_; - if (unFinishedPart == 0) { - // all parts has finished - LOG(INFO) << "Handle last part: " << partId << " vertex/edge/index data in space id " - << spaceId_ << " finished."; - } else { - pool_->add(std::bind(&UpgraderSpace::runPartV3, this)); - } - } else { - LOG(INFO) << "Handle vertex/edge/index of parts data in space id " << spaceId_ << " finished"; - } -} -void UpgraderSpace::doProcessV3() { - LOG(INFO) << "Start to handle data in space id " << spaceId_; - // Parallel process part - auto partConcurrency = std::min(static_cast(FLAGS_max_concurrent_parts), parts_.size()); - LOG(INFO) << "Max concurrent parts: " << partConcurrency; - unFinishedPart_ = parts_.size(); - - LOG(INFO) << "Start to handle vertex/edge/index of parts data in space id " << spaceId_; - for (size_t i = 0; i < partConcurrency; ++i) { - pool_->add(std::bind(&UpgraderSpace::runPartV3, this)); - } - - while (unFinishedPart_ != 0) { - sleep(10); - } - - if (ingest_sst_file_.size() != 0) { - auto code = readEngine_->ingest(ingest_sst_file_, true); - if (code != ::nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(FATAL) << "Faild upgrade 2:3 when ingest sst file:" << static_cast(code); - } - } - readEngine_->put(NebulaKeyUtils::dataVersionKey(), NebulaKeyUtilsV3::dataVersionValue()); -} -std::vector UpgraderSpace::indexVertexKeys( - PartitionID partId, - VertexID& vId, - RowReader* reader, - std::shared_ptr index, - const meta::SchemaProviderIf* latestSchema) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), latestSchema); - if (!values.ok()) { - return {}; - } - return IndexKeyUtils::vertexIndexKeys( - spaceVidLen_, partId, index->get_index_id(), vId, std::move(values).value()); -} - -void UpgraderSpace::encodeEdgeValue(PartitionID partId, - RowReader* reader, - const meta::NebulaSchemaProvider* schema, - std::string& newkey, - VertexID& svId, - EdgeType type, - EdgeRanking rank, - VertexID& dstId, - std::vector& data) { - // Get all returned field name - auto& fieldNames = edgeFieldName_[std::abs(type)]; - - auto ret = encodeRowVal(reader, schema, fieldNames); - if (ret.empty()) { - return; - } - data.emplace_back(std::move(newkey), ret); - - if (type <= 0) { - return; - } - - // encode v2 index value - auto it = edgeIndexes_.find(type); - if (it != edgeIndexes_.end()) { - // Use new RowReader - auto nReader = RowReaderWrapper::getEdgePropReader(schemaMan_, spaceId_, type, ret); - if (nReader == nullptr) { - LOG(ERROR) << "Bad format row: space id " << spaceId_ << " edgetype " << type - << " srcVertexId " << svId << " rank " << rank << " dstVertexId " << dstId; - return; - } - for (auto& index : it->second) { - auto newIndexKeys = indexEdgeKeys(partId, nReader.get(), svId, rank, dstId, index, schema); - for (auto& newIndexKey : newIndexKeys) { - data.emplace_back(std::move(newIndexKey), ""); - } - } - } -} - -std::vector UpgraderSpace::indexEdgeKeys( - PartitionID partId, - RowReader* reader, - VertexID& svId, - EdgeRanking rank, - VertexID& dstId, - std::shared_ptr index, - const meta::SchemaProviderIf* latestSchema) { - auto values = IndexKeyUtils::collectIndexValues(reader, index.get(), latestSchema); - if (!values.ok()) { - return {}; - } - return IndexKeyUtils::edgeIndexKeys( - spaceVidLen_, partId, index->get_index_id(), svId, rank, dstId, std::move(values).value()); -} - -void UpgraderSpace::doCompaction() { - LOG(INFO) << "Path " << dstPath_ << " space id " << spaceId_ << " compaction begin"; - auto ret = writeEngine_->compact(); - if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(ERROR) << "Path " << dstPath_ << " space id " << spaceId_ << " compaction failed!"; - } - LOG(INFO) << "Path " << dstPath_ << " space id " << spaceId_ << " compaction success!"; -} - -bool UpgraderSpace::copyWal() { - LOG(INFO) << "Copy space id " << entry_ << " wal file begin"; - // Get source wal directory - auto srcPath = - folly::stringPrintf("%s/%s/%s/%s", srcPath_.c_str(), "nebula", entry_.c_str(), "wal"); - if (!fs::FileUtils::exist(srcPath)) { - LOG(ERROR) << "Source data wal path " << srcPath << " not exists!"; - return false; - } - // Get destination wal directory - auto dstPath = folly::stringPrintf("%s/%s/%s", dstPath_.c_str(), "nebula", entry_.c_str()); - if (!fs::FileUtils::exist(dstPath)) { - LOG(ERROR) << "Destination data wal path " << dstPath << " not exists!"; - return false; - } - dstPath = fs::FileUtils::joinPath(dstPath, "wal"); - if (!fs::FileUtils::makeDir(dstPath)) { - LOG(FATAL) << "makeDir " << dstPath << " failed"; - } - - auto partDirs = fs::FileUtils::listAllDirsInDir(srcPath.c_str()); - for (size_t i = 0; i < partDirs.size(); i++) { - // In general, there are only two wal files left for each part - auto files = fs::FileUtils::listAllFilesInDir( - folly::stringPrintf("%s/%s", srcPath.c_str(), partDirs[i].c_str()).c_str()); - // If the number of wal files is greater than two, find the latest two wal - // files - auto walNum = files.size(); - if (walNum > 2) { - std::sort(files.begin(), files.end()); - auto newestFile = files[walNum - 2]; - auto latestFile = files[walNum - 1]; - files.resize(2); - files[0] = newestFile; - files[1] = latestFile; - } - - for (const auto& file : files) { - std::fstream srcF( - folly::stringPrintf("%s/%s/%s", srcPath.c_str(), partDirs[i].c_str(), file.c_str()), - std::ios::binary | std::ios::in); - auto dstwalpart = folly::stringPrintf("%s/%s", dstPath.c_str(), partDirs[i].c_str()); - if (!fs::FileUtils::makeDir(dstwalpart)) { - LOG(FATAL) << "makeDir " << dstwalpart << " failed"; - } - std::fstream destF(folly::stringPrintf("%s/%s", dstwalpart.c_str(), file.c_str()), - std::ios::binary | std::ios::out); - destF << srcF.rdbuf(); - destF.close(); - srcF.close(); - } - } - return true; -} - -Status DbUpgrader::init(meta::MetaClient* mclient, - meta::ServerBasedSchemaManager* sMan, - meta::IndexManager* iMan, - const std::string& srcPath, - const std::string& dstPath) { - metaClient_ = mclient; - schemaMan_ = sMan; - indexMan_ = iMan; - srcPath_ = srcPath; - dstPath_ = dstPath; - pool_ = std::make_unique(FLAGS_max_concurrent_spaces); - return listSpace(); -} - -Status DbUpgrader::listSpace() { - // from srcPath_ to srcPath_/nebula - auto path = fs::FileUtils::joinPath(srcPath_, "nebula"); - if (!fs::FileUtils::exist(path)) { - LOG(ERROR) << "Source data path " << srcPath_ << " not exists!"; - return Status::Error("Db path '%s' not exists.", srcPath_.c_str()); - } - - if (!fs::FileUtils::exist(dstPath_)) { - LOG(ERROR) << "Destination data path " << dstPath_ << " not exists!"; - return Status::Error("Db path '%s' not exists.", dstPath_.c_str()); - } - subDirs_ = fs::FileUtils::listAllDirsInDir(path.c_str()); - return Status::OK(); -} - -void DbUpgrader::run() { - LOG(INFO) << "Upgrade from path " << srcPath_ << " to path " << dstPath_ - << " in DbUpgrader run begin"; - // Get all the directories in the data directory, - // each directory name is spaceId, traverse each directory - std::vector> upgraderSpaces; - for (auto& entry : subDirs_) { - auto it = std::make_unique(); - - // When the init space fails, ignore to upgrade this space - auto ret = it->init(metaClient_, schemaMan_, indexMan_, srcPath_, dstPath_, entry); - if (!ret.ok()) { - LOG(WARNING) << "Upgrade from path " << srcPath_ << " space id " << entry << " to path " - << dstPath_ << " init failed"; - LOG(WARNING) << "Ignore upgrade " << srcPath_ << " space id " << entry; - } else { - upgraderSpaces.emplace_back(std::move(it)); - } - } - - unFinishedSpace_ = upgraderSpaces.size(); - for (size_t i = 0; i < upgraderSpaces.size(); i++) { - spaceQueue_.add(upgraderSpaces[i].get()); - } - - // Parallel process space - auto spaceConcurrency = - std::min(static_cast(FLAGS_max_concurrent_spaces), upgraderSpaces.size()); - LOG(INFO) << "Max concurrent spaces: " << spaceConcurrency; - - for (size_t i = 0; i < spaceConcurrency; ++i) { - pool_->add(std::bind(&DbUpgrader::doSpace, this)); - } - - while (unFinishedSpace_ != 0) { - sleep(10); - } - - LOG(INFO) << "Upgrade from path " << srcPath_ << " to path " << dstPath_ - << " in DbUpgrader run end"; -} - -void DbUpgrader::doSpace() { - std::chrono::milliseconds take_dura{10}; - if (auto UpSpace = spaceQueue_.try_take_for(take_dura)) { - auto upgraderSpaceIter = *UpSpace; - LOG(INFO) << "Upgrade from path " << upgraderSpaceIter->srcPath_ << " space id " - << upgraderSpaceIter->entry_ << " to path " << upgraderSpaceIter->dstPath_ - << " begin"; - if (FLAGS_upgrade_version == "1:2") { - upgraderSpaceIter->doProcessV1(); - } else if (FLAGS_upgrade_version == "2RC:2") { - upgraderSpaceIter->doProcessV2(); - } else if (FLAGS_upgrade_version == "2:3") { - upgraderSpaceIter->doProcessV3(); - } else { - LOG(FATAL) << "error upgrade version " << FLAGS_upgrade_version; - } - - auto ret = upgraderSpaceIter->copyWal(); - if (!ret) { - LOG(ERROR) << "Copy space id " << upgraderSpaceIter->entry_ << " wal file failed"; - } else { - LOG(INFO) << "Copy space id " << upgraderSpaceIter->entry_ << " wal file success"; - } - - if (FLAGS_compactions) { - upgraderSpaceIter->doCompaction(); - } - - auto unFinishedSpace = --unFinishedSpace_; - if (unFinishedSpace == 0) { - // all spaces has finished - LOG(INFO) << "Upgrade last space: " << upgraderSpaceIter->entry_ << " from " - << upgraderSpaceIter->srcPath_ << " to path " << upgraderSpaceIter->dstPath_ - << " end"; - } else { - pool_->add(std::bind(&DbUpgrader::doSpace, this)); - } - } else { - LOG(INFO) << "Upgrade from path " << srcPath_ << " to path " << dstPath_ << " end"; - } -} - -} // namespace storage -} // namespace nebula diff --git a/src/tools/db-upgrade/DbUpgrader.h b/src/tools/db-upgrade/DbUpgrader.h deleted file mode 100644 index b66543aa882..00000000000 --- a/src/tools/db-upgrade/DbUpgrader.h +++ /dev/null @@ -1,223 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef TOOLS_DBUPGRADE_DBUPGRADER_H_ -#define TOOLS_DBUPGRADE_DBUPGRADER_H_ - -#include -#include -#include - -#include "clients/meta/MetaClient.h" -#include "codec/RowReaderWrapper.h" -#include "codec/RowWriterV2.h" -#include "common/base/Base.h" -#include "common/base/Status.h" -#include "common/meta/ServerBasedIndexManager.h" -#include "common/meta/ServerBasedSchemaManager.h" -#include "kvstore/RocksEngine.h" - -DECLARE_string(src_db_path); -DECLARE_string(dst_db_path); -DECLARE_string(upgrade_meta_server); -DECLARE_uint32(write_batch_num); -DECLARE_string(upgrade_version); -DECLARE_bool(compactions); -DECLARE_uint32(max_concurrent_parts); -DECLARE_uint32(max_concurrent_spaces); - -namespace nebula { -namespace storage { - -// Upgrade a space of data path in storage conf -class UpgraderSpace { - public: - UpgraderSpace() = default; - - ~UpgraderSpace() { - if (pool_) { - pool_->join(); - } - } - - Status init(meta::MetaClient* mclient, - meta::ServerBasedSchemaManager* sMan, - meta::IndexManager* iMan, - const std::string& srcPath, - const std::string& dstPath, - const std::string& entry); - - // Process v1 data and upgrade to v2 Ga - void doProcessV1(); - - // Processing v2 Rc data upgrade to v2 Ga - void doProcessV2(); - - // Processing v2 Ga data upgrade to v3 - void doProcessV3(); - - // Perform manual compact - void doCompaction(); - - // Copy the latest wal file under each part, two at most - bool copyWal(); - - private: - Status initSpace(const std::string& spaceId); - - Status buildSchemaAndIndex(); - - bool isValidVidLen(VertexID srcVId, VertexID dstVId = ""); - - void encodeVertexValue(PartitionID partId, - RowReader* reader, - const meta::NebulaSchemaProvider* schema, - std::string& newkey, - VertexID& strVid, - TagID tagId, - std::vector& data); - - // Used for vertex and edge - std::string encodeRowVal(const RowReader* reader, - const meta::NebulaSchemaProvider* schema, - std::vector& fieldName); - - std::vector indexVertexKeys(PartitionID partId, - VertexID& vId, - RowReader* reader, - std::shared_ptr index, - const meta::SchemaProviderIf* latestSchema); - - void encodeEdgeValue(PartitionID partId, - RowReader* reader, - const meta::NebulaSchemaProvider* schema, - std::string& newkey, - VertexID& svId, - EdgeType type, - EdgeRanking rank, - VertexID& dstId, - std::vector& data); - - std::vector indexEdgeKeys(PartitionID partId, - RowReader* reader, - VertexID& svId, - EdgeRanking rank, - VertexID& dstId, - std::shared_ptr index, - const meta::SchemaProviderIf* latestSchema); - - WriteResult convertValue(const meta::NebulaSchemaProvider* newSchema, - const meta::SchemaProviderIf* oldSchema, - std::string& name, - Value& val); - void runPartV1(); - - void runPartV2(); - - void runPartV3(); - - public: - // Source data path - std::string srcPath_; - // Destination data path - std::string dstPath_; - std::string entry_; - - private: - meta::MetaClient* metaClient_; - meta::ServerBasedSchemaManager* schemaMan_; - meta::IndexManager* indexMan_; - - // The following variables are space level - GraphSpaceID spaceId_; - int32_t spaceVidLen_; - std::string spaceName_; - std::vector parts_; - std::unique_ptr readEngine_; - std::unique_ptr writeEngine_; - - // Get all tag newest schema in space - std::unordered_map>> - tagSchemas_; - - // tagId -> All field names in newest schema - std::unordered_map> tagFieldName_; - - // tagId -> all indexes of this tag - std::unordered_map>> - tagIndexes_; - - // Get all edge newest schema in space - std::unordered_map>> - edgeSchemas_; - - // edgetype -> all field name in newest schema - std::unordered_map> edgeFieldName_; - - // edgetype -> all indexes of this edgetype - std::unordered_map>> - edgeIndexes_; - - // for parallel parts - std::unique_ptr pool_{nullptr}; - - folly::UnboundedBlockingQueue partQueue_; - - std::atomic unFinishedPart_; - - std::mutex ingest_sst_file_mut_; - std::vector ingest_sst_file_; -}; - -// Upgrade one data path in storage conf -class DbUpgrader { - public: - DbUpgrader() = default; - - ~DbUpgrader() { - if (pool_) { - pool_->join(); - } - } - - Status init(meta::MetaClient* mclient, - meta::ServerBasedSchemaManager* sMan, - meta::IndexManager* iMan, - const std::string& srcPath, - const std::string& dstPath); - - void run(); - - private: - // Get all string spaceId - Status listSpace(); - - void doProcessAllTagsAndEdges(); - - void doSpace(); - - private: - meta::MetaClient* metaClient_; - meta::ServerBasedSchemaManager* schemaMan_; - meta::IndexManager* indexMan_; - // Source data path - std::string srcPath_; - - // Destination data path - std::string dstPath_; - std::vector subDirs_; - - // for parallel spaces - std::unique_ptr pool_{nullptr}; - - folly::UnboundedBlockingQueue spaceQueue_; - - std::atomic unFinishedSpace_; -}; - -} // namespace storage -} // namespace nebula - -#endif // TOOLS_DBUPGRADE_DBUPGRADER_H_ diff --git a/src/tools/db-upgrade/DbUpgraderTool.cpp b/src/tools/db-upgrade/DbUpgraderTool.cpp deleted file mode 100644 index b4e05cd143b..00000000000 --- a/src/tools/db-upgrade/DbUpgraderTool.cpp +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "common/base/Base.h" -#include "kvstore/RocksEngineConfig.h" -#include "tools/db-upgrade/DbUpgrader.h" - -void printHelp() { - fprintf( - stderr, - R"( ./db_upgrade --src_db_path= --dst_db_path= --upgrade_meta_server= --upgrade_version=2:3 - -desc: - This tool is used to upgrade data from nebula 2.0GA to 3.0 - -required: - --src_db_path= - Source data path to the rocksdb data directory. - This is an absolute path, multi paths should be split by comma. - If old nebula was installed in /usr/local/nebula, - the db_path would be /usr/local/nebula/data/storage - Default: "" - - --dst_db_path= - Destination data path to the rocksdb data directory. - This is an absolute path, multi paths should be split by comma. - If new nebula was installed in /usr/local/nebula_new, - the db_path would be /usr/local/nebulav_new/data/storage - Default: "" - - note: - The number of paths in src_db_path is equal to the number of paths in dst_db_path, and - src_db_path and dst_db_path must be different. - For 2.0GA to 3.0, dst_db_path is useless. - - --upgrade_meta_server= - A list of meta severs' ip:port separated by comma. - Default: 127.0.0.1:45500 - - --upgrade_version=<2:3> - This tool can only upgrade 2.0GA. - 2:3 upgrade the data from 2.0GA to 3.0 - Default: "" - - optional: - --write_batch_num= - The size of the batch written to rocksdb. - Default: 100 - - --compactions= - When the data upgrade finished, whether to compact all data. - Default: true - - --max_concurrent_parts - Maximum number of concurrent parts allowed. - Default: 10 - - --max_concurrent_spaces - Maximum number of concurrent spaces allowed. - Default: 5 -)"); -} - -void printParams() { - std::cout << "===========================PARAMS============================\n"; - std::cout << "meta server: " << FLAGS_upgrade_meta_server << "\n"; - std::cout << "source data path: " << FLAGS_src_db_path << "\n"; - std::cout << "destination data path: " << FLAGS_dst_db_path << "\n"; - std::cout << "The size of the batch written: " << FLAGS_write_batch_num << "\n"; - std::cout << "upgrade data from version: " << FLAGS_upgrade_version << "\n"; - std::cout << "whether to compact all data: " << (FLAGS_compactions == true ? "true" : "false") - << "\n"; - std::cout << "maximum number of concurrent parts allowed:" << FLAGS_max_concurrent_parts << "\n"; - std::cout << "maximum number of concurrent spaces allowed: " << FLAGS_max_concurrent_spaces - << "\n"; - std::cout << "===========================PARAMS============================\n\n"; -} - -int main(int argc, char* argv[]) { - // When begin to upgrade the data, close compaction - // When upgrade finished, perform compaction. - FLAGS_rocksdb_column_family_options = R"({ - "disable_auto_compactions":"true", - "write_buffer_size":"134217728", - "max_write_buffer_number":"12", - "max_bytes_for_level_base":"268435456", - "level0_slowdown_writes_trigger":"999999", - "level0_stop_writes_trigger":"999999", - "soft_pending_compaction_bytes_limit":"137438953472", - "hard_pending_compaction_bytes_limit":"274877906944" - })"; - - FLAGS_rocksdb_db_options = R"({ - "max_background_jobs":"10", - "max_subcompactions":"10" - })"; - - if (argc == 1) { - printHelp(); - return EXIT_FAILURE; - } else { - folly::init(&argc, &argv, true); - } - - google::SetStderrLogging(google::INFO); - - printParams(); - - // Handle arguments - LOG(INFO) << "Prepare phase begin"; - if (FLAGS_src_db_path.empty() || FLAGS_dst_db_path.empty()) { - LOG(ERROR) << "Source data path or destination data path should be not empty."; - return EXIT_FAILURE; - } - - std::vector srcPaths; - folly::split(",", FLAGS_src_db_path, srcPaths, true); - std::transform(srcPaths.begin(), srcPaths.end(), srcPaths.begin(), [](auto& p) { - return folly::trimWhitespace(p).str(); - }); - if (srcPaths.empty()) { - LOG(ERROR) << "Bad source data path format: " << FLAGS_src_db_path; - return EXIT_FAILURE; - } - - std::vector dstPaths; - folly::split(",", FLAGS_dst_db_path, dstPaths, true); - std::transform(dstPaths.begin(), dstPaths.end(), dstPaths.begin(), [](auto& p) { - return folly::trimWhitespace(p).str(); - }); - if (dstPaths.empty()) { - LOG(ERROR) << "Bad destination data path format: " << FLAGS_dst_db_path; - return EXIT_FAILURE; - } - - if (srcPaths.size() != dstPaths.size()) { - LOG(ERROR) << "The size of source data paths is not equal the " - << "size of destination data paths."; - return EXIT_FAILURE; - } - - auto addrs = nebula::network::NetworkUtils::toHosts(FLAGS_upgrade_meta_server); - if (!addrs.ok()) { - LOG(ERROR) << "Get meta host address failed " << FLAGS_upgrade_meta_server; - return EXIT_FAILURE; - } - - auto ioExecutor = std::make_shared(1); - nebula::meta::MetaClientOptions options; - options.skipConfig_ = true; - auto metaClient = - std::make_unique(ioExecutor, std::move(addrs.value()), options); - CHECK_NOTNULL(metaClient); - if (!metaClient->waitForMetadReady(1)) { - LOG(ERROR) << "Meta is not ready: " << FLAGS_upgrade_meta_server; - return EXIT_FAILURE; - } - - auto schemaMan = nebula::meta::ServerBasedSchemaManager::create(metaClient.get()); - auto indexMan = nebula::meta::ServerBasedIndexManager::create(metaClient.get()); - CHECK_NOTNULL(schemaMan); - CHECK_NOTNULL(indexMan); - - std::vector versions = {"2:3"}; - if (std::find(versions.begin(), versions.end(), FLAGS_upgrade_version) == versions.end()) { - LOG(ERROR) << "Flag upgrade_version : " << FLAGS_upgrade_version; - return EXIT_FAILURE; - } - LOG(INFO) << "Prepare phase end"; - - // Upgrade data - LOG(INFO) << "Upgrade phase begin"; - - // The data path in storage conf is generally one, not too many. - // So there is no need to control the number of threads here. - std::vector threads; - for (size_t i = 0; i < srcPaths.size(); i++) { - threads.emplace_back(std::thread([mclient = metaClient.get(), - sMan = schemaMan.get(), - iMan = indexMan.get(), - srcPath = srcPaths[i], - dstPath = dstPaths[i]] { - LOG(INFO) << "Upgrade from path " << srcPath << " to path " << dstPath << " begin"; - nebula::storage::DbUpgrader upgrader; - auto ret = upgrader.init(mclient, sMan, iMan, srcPath, dstPath); - if (!ret.ok()) { - LOG(ERROR) << "Upgrader from path " << srcPath << " to path " << dstPath << " init failed."; - return; - } - upgrader.run(); - LOG(INFO) << "Upgrade from path " << srcPath << " to path " << dstPath << " end"; - })); - } - - // Wait for all threads to finish - for (auto& t : threads) { - t.join(); - } - - LOG(INFO) << "Upgrade phase end"; - return 0; -} diff --git a/src/tools/db-upgrade/NebulaKeyUtilsV1.cpp b/src/tools/db-upgrade/NebulaKeyUtilsV1.cpp deleted file mode 100644 index 9948422a709..00000000000 --- a/src/tools/db-upgrade/NebulaKeyUtilsV1.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright (c) 2018 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "tools/db-upgrade/NebulaKeyUtilsV1.h" - -namespace nebula { - -// static -std::string NebulaKeyUtilsV1::indexPrefix(PartitionID partId, IndexID indexId) { - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kIndex); - std::string key; - key.reserve(sizeof(PartitionID) + sizeof(IndexID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&indexId), sizeof(IndexID)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::vertexPrefix(PartitionID partId, VertexID vId, TagID tagId) { - tagId &= kTagMaskSet; - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kData); - - std::string key; - key.reserve(kVertexLen); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&vId), sizeof(VertexID)) - .append(reinterpret_cast(&tagId), sizeof(TagID)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::edgePrefix(PartitionID partId, VertexID srcId, EdgeType type) { - type |= kEdgeMaskSet; - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kData); - - std::string key; - key.reserve(sizeof(PartitionID) + sizeof(VertexID) + sizeof(EdgeType)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&srcId), sizeof(VertexID)) - .append(reinterpret_cast(&type), sizeof(EdgeType)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::edgePrefix( - PartitionID partId, VertexID srcId, EdgeType type, EdgeRanking rank, VertexID dstId) { - type |= kEdgeMaskSet; - int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kData); - std::string key; - key.reserve(sizeof(PartitionID) + sizeof(VertexID) + sizeof(EdgeType) + sizeof(VertexID) + - sizeof(EdgeRanking)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&srcId), sizeof(VertexID)) - .append(reinterpret_cast(&type), sizeof(EdgeType)) - .append(reinterpret_cast(&rank), sizeof(EdgeRanking)) - .append(reinterpret_cast(&dstId), sizeof(VertexID)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::prefix(PartitionID partId) { - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kData); - std::string key; - key.reserve(sizeof(PartitionID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::snapshotPrefix(PartitionID partId) { - // snapshot of meta would be all key-value pairs - if (partId == 0) { - return ""; - } - return prefix(partId); -} - -// static -std::string NebulaKeyUtilsV1::vertexPrefix(PartitionID partId, VertexID vId) { - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kData); - std::string key; - key.reserve(sizeof(PartitionID) + sizeof(VertexID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&vId), sizeof(VertexID)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::edgePrefix(PartitionID partId, VertexID vId) { - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV1::kData); - std::string key; - key.reserve(sizeof(PartitionID) + sizeof(VertexID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&vId), sizeof(VertexID)); - return key; -} - -// static -std::string NebulaKeyUtilsV1::systemPrefix() { - int8_t type = static_cast(NebulaKeyTypeV1::kSystem); - std::string key; - key.reserve(sizeof(int8_t)); - key.append(reinterpret_cast(&type), sizeof(int8_t)); - return key; -} - -} // namespace nebula diff --git a/src/tools/db-upgrade/NebulaKeyUtilsV1.h b/src/tools/db-upgrade/NebulaKeyUtilsV1.h deleted file mode 100644 index 3226f34b626..00000000000 --- a/src/tools/db-upgrade/NebulaKeyUtilsV1.h +++ /dev/null @@ -1,252 +0,0 @@ -/* Copyright (c) 2018 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef TOOLS_DBUPGRADE_NEBULAKEYUTILSV1_H_ -#define TOOLS_DBUPGRADE_NEBULAKEYUTILSV1_H_ - -#include "common/utils/Types.h" - -namespace nebula { - -enum class NebulaKeyTypeV1 : uint32_t { - kData = 0x00000001, - kIndex = 0x00000002, - kUUID = 0x00000003, - kSystem = 0x00000004, -}; - -/** - * This class supply some utils for transition between Vertex/Edge and key in - * kvstore. - * */ -class NebulaKeyUtilsV1 final { - public: - ~NebulaKeyUtilsV1() = default; - - using VertexID = int64_t; - - static std::string indexPrefix(PartitionID partId, IndexID indexId); - - static std::string vertexPrefix(PartitionID partId, VertexID vId, TagID tagId); - - static std::string edgePrefix(PartitionID partId, VertexID srcId, EdgeType type); - - static std::string vertexPrefix(PartitionID partId, VertexID vId); - - static std::string edgePrefix(PartitionID partId, VertexID vId); - - static std::string edgePrefix( - PartitionID partId, VertexID srcId, EdgeType type, EdgeRanking rank, VertexID dstId); - - static std::string systemPrefix(); - - static std::string prefix(PartitionID partId); - - static std::string snapshotPrefix(PartitionID partId); - - static PartitionID getPart(const folly::StringPiece& rawKey) { - return readInt(rawKey.data(), sizeof(PartitionID)) >> 8; - } - - static bool isVertex(const folly::StringPiece& rawKey) { - if (rawKey.size() != kVertexLen) { - return false; - } - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV1)); - auto type = readInt(rawKey.data(), len) & kTypeMask; - if (static_cast(NebulaKeyTypeV1::kData) != type) { - return false; - } - auto offset = sizeof(PartitionID) + sizeof(VertexID); - TagID tagId = readInt(rawKey.data() + offset, sizeof(TagID)); - return !(tagId & kTagEdgeMask); - } - - static VertexID getVertexId(const folly::StringPiece& rawKey) { - CHECK_EQ(rawKey.size(), kVertexLen); - auto offset = sizeof(PartitionID); - return readInt(rawKey.data() + offset, sizeof(VertexID)); - } - - static TagID getTagId(const folly::StringPiece& rawKey) { - // CHECK_EQ(rawKey.size(), kVertexLen); - if (rawKey.size() != kVertexLen) { - std::stringstream msg; - msg << " rawKey.size() != kVertexLen." - << "\nrawKey.size()=" << rawKey.size() << "\nkVertexLen=" << kVertexLen << "\nhexDump:\n" - << folly::hexDump(rawKey.data(), rawKey.size()); - LOG(FATAL) << msg.str(); - } - auto offset = sizeof(PartitionID) + sizeof(VertexID); - return readInt(rawKey.data() + offset, sizeof(TagID)); - } - - static bool isEdge(const folly::StringPiece& rawKey) { - if (rawKey.size() != kEdgeLen) { - return false; - } - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV1)); - auto type = readInt(rawKey.data(), len) & kTypeMask; - if (static_cast(NebulaKeyTypeV1::kData) != type) { - return false; - } - auto offset = sizeof(PartitionID) + sizeof(VertexID); - EdgeType etype = readInt(rawKey.data() + offset, sizeof(EdgeType)); - return etype & kTagEdgeMask; - } - - static bool isSystemCommit(const folly::StringPiece& rawKey) { - if (rawKey.size() != kSystemLen) { - return false; - } - auto position = rawKey.data() + sizeof(PartitionID); - auto len = sizeof(NebulaSystemKeyType); - auto type = readInt(position, len); - return static_cast(NebulaSystemKeyType::kSystemCommit) == type; - } - - static bool isSystemPart(const folly::StringPiece& rawKey) { - if (rawKey.size() != kSystemLen) { - return false; - } - auto position = rawKey.data() + sizeof(PartitionID); - auto len = sizeof(NebulaSystemKeyType); - auto type = readInt(position, len); - return static_cast(NebulaSystemKeyType::kSystemPart) == type; - } - - static VertexID getSrcId(const folly::StringPiece& rawKey) { - CHECK_EQ(rawKey.size(), kEdgeLen); - return readInt(rawKey.data() + sizeof(PartitionID), sizeof(VertexID)); - } - - static VertexID getDstId(const folly::StringPiece& rawKey) { - CHECK_EQ(rawKey.size(), kEdgeLen); - auto offset = sizeof(PartitionID) + sizeof(VertexID) + sizeof(EdgeType) + sizeof(EdgeRanking); - return readInt(rawKey.data() + offset, sizeof(VertexID)); - } - - static EdgeType getEdgeType(const folly::StringPiece& rawKey) { - CHECK_EQ(rawKey.size(), kEdgeLen); - auto offset = sizeof(PartitionID) + sizeof(VertexID); - EdgeType type = readInt(rawKey.data() + offset, sizeof(EdgeType)); - return type > 0 ? type & kTagEdgeValueMask : type; - } - - static EdgeRanking getRank(const folly::StringPiece& rawKey) { - CHECK_EQ(rawKey.size(), kEdgeLen); - auto offset = sizeof(PartitionID) + sizeof(VertexID) + sizeof(EdgeType); - return readInt(rawKey.data() + offset, sizeof(EdgeRanking)); - } - - static int64_t getVersion(const folly::StringPiece& rawKey) { - CHECK(isVertex(rawKey) || isEdge(rawKey)); - auto offset = rawKey.size() - sizeof(int64_t); - return readInt(rawKey.data() + offset, sizeof(int64_t)); - } - - static IndexID getIndexId(const folly::StringPiece& rawKey) { - CHECK_GT(rawKey.size(), kIndexLen); - auto offset = sizeof(PartitionID); - return readInt(rawKey.data() + offset, sizeof(IndexID)); - } - - template - static typename std::enable_if::value, T>::type readInt(const char* data, - int32_t len) { - CHECK_GE(len, sizeof(T)); - return *reinterpret_cast(data); - } - - static bool isDataKey(const folly::StringPiece& key) { - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV1)); - auto type = readInt(key.data(), len) & kTypeMask; - return static_cast(NebulaKeyTypeV1::kData) == type; - } - - static bool isIndexKey(const folly::StringPiece& key) { - if (key.size() < kIndexLen) { - return false; - } - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV1)); - auto type = readInt(key.data(), len) & kTypeMask; - return static_cast(NebulaKeyTypeV1::kIndex) == type; - } - - static bool isUUIDKey(const folly::StringPiece& key) { - auto type = readInt(key.data(), sizeof(int32_t)) & kTypeMask; - return static_cast(NebulaKeyTypeV1::kUUID) == type; - } - - static folly::StringPiece keyWithNoVersion(const folly::StringPiece& rawKey) { - // TODO(heng) We should change the method if varint data version supported. - return rawKey.subpiece(0, rawKey.size() - sizeof(int64_t)); - } - - static VertexID getIndexVertexID(const folly::StringPiece& rawKey) { - CHECK_GE(rawKey.size(), kVertexIndexLen); - auto offset = rawKey.size() - sizeof(VertexID); - return *reinterpret_cast(rawKey.data() + offset); - } - - static VertexID getIndexSrcId(const folly::StringPiece& rawKey) { - CHECK_GE(rawKey.size(), kEdgeIndexLen); - auto offset = rawKey.size() - sizeof(VertexID) * 2 - sizeof(EdgeRanking); - return readInt(rawKey.data() + offset, sizeof(VertexID)); - } - - static VertexID getIndexDstId(const folly::StringPiece& rawKey) { - CHECK_GE(rawKey.size(), kEdgeIndexLen); - auto offset = rawKey.size() - sizeof(VertexID); - return readInt(rawKey.data() + offset, sizeof(VertexID)); - } - - static EdgeRanking getIndexRank(const folly::StringPiece& rawKey) { - CHECK_GE(rawKey.size(), kEdgeIndexLen); - auto offset = rawKey.size() - sizeof(VertexID) - sizeof(EdgeRanking); - return readInt(rawKey.data() + offset, sizeof(EdgeRanking)); - } - - private: - NebulaKeyUtilsV1() = delete; - - private: - static constexpr int32_t kVertexLen = - sizeof(PartitionID) + sizeof(VertexID) + sizeof(TagID) + sizeof(TagVersion); - - static constexpr int32_t kEdgeLen = sizeof(PartitionID) + sizeof(VertexID) + sizeof(EdgeType) + - sizeof(VertexID) + sizeof(EdgeRanking) + sizeof(EdgeVersion); - - static constexpr int32_t kVertexIndexLen = - sizeof(PartitionID) + sizeof(IndexID) + sizeof(VertexID); - - static constexpr int32_t kEdgeIndexLen = - sizeof(PartitionID) + sizeof(IndexID) + sizeof(VertexID) * 2 + sizeof(EdgeRanking); - - static constexpr int32_t kIndexLen = std::min(kVertexIndexLen, kEdgeIndexLen); - - static constexpr int32_t kSystemLen = sizeof(PartitionID) + sizeof(NebulaSystemKeyType); - - // The partition id offset in 4 Bytes - static constexpr uint8_t kPartitionOffset = 8; - - // The key type bits Mask - // See KeyType enum - static constexpr uint32_t kTypeMask = 0x000000FF; - - // The most significant bit is sign bit, tag is always 0 - // The second most significant bit is tag/edge type bit Mask - // 0 for Tag, 1 for Edge - static constexpr uint32_t kTagEdgeMask = 0x40000000; - // For extract Tag/Edge value - static constexpr uint32_t kTagEdgeValueMask = ~kTagEdgeMask; - // Write edge by |= 0x40000000 - static constexpr uint32_t kEdgeMaskSet = kTagEdgeMask; - // Write Tag by &= 0xbfffffff - static constexpr uint32_t kTagMaskSet = ~kTagEdgeMask; -}; - -} // namespace nebula -#endif // TOOLS_DBUPGRADE_NEBULAKEYUTILSV1_H_ diff --git a/src/tools/db-upgrade/NebulaKeyUtilsV2.cpp b/src/tools/db-upgrade/NebulaKeyUtilsV2.cpp deleted file mode 100644 index 8a1a0f429c6..00000000000 --- a/src/tools/db-upgrade/NebulaKeyUtilsV2.cpp +++ /dev/null @@ -1,225 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "tools/db-upgrade/NebulaKeyUtilsV2.h" - -namespace nebula { - -// static -bool NebulaKeyUtilsV2::isValidVidLen(size_t vIdLen, VertexID srcVId, VertexID dstVId) { - if (srcVId.size() > vIdLen || dstVId.size() > vIdLen) { - return false; - } - return true; -} - -// static -std::string NebulaKeyUtilsV2::tagKey( - size_t vIdLen, PartitionID partId, VertexID vId, TagID tagId, TagVersion tv) { - CHECK_GE(vIdLen, vId.size()); - tagId &= kTagMaskSet; - int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - - std::string key; - key.reserve(kVertexLen + vIdLen); - key.append(reinterpret_cast(&item), sizeof(int32_t)) - .append(vId.data(), vId.size()) - .append(vIdLen - vId.size(), '\0') - .append(reinterpret_cast(&tagId), sizeof(TagID)) - .append(reinterpret_cast(&tv), sizeof(TagVersion)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::edgeKey(size_t vIdLen, - PartitionID partId, - VertexID srcId, - EdgeType type, - EdgeRanking rank, - VertexID dstId, - EdgeVersion ev) { - CHECK_GE(vIdLen, srcId.size()); - CHECK_GE(vIdLen, dstId.size()); - type |= kEdgeMaskSet; - int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - - std::string key; - key.reserve(kEdgeLen + (vIdLen << 1)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(srcId.data(), srcId.size()) - .append(vIdLen - srcId.size(), '\0') - .append(reinterpret_cast(&type), sizeof(EdgeType)) - .append(reinterpret_cast(&rank), sizeof(EdgeRanking)) - .append(dstId.data(), dstId.size()) - .append(vIdLen - dstId.size(), '\0') - .append(reinterpret_cast(&ev), sizeof(EdgeVersion)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::systemCommitKey(PartitionID partId) { - int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kSystem); - uint32_t type = static_cast(NebulaSystemKeyType::kSystemCommit); - std::string key; - key.reserve(kSystemLen); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&type), sizeof(NebulaSystemKeyType)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::systemPartKey(PartitionID partId) { - uint32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kSystem); - uint32_t type = static_cast(NebulaSystemKeyType::kSystemPart); - std::string key; - key.reserve(kSystemLen); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(reinterpret_cast(&type), sizeof(NebulaSystemKeyType)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::kvKey(PartitionID partId, const folly::StringPiece& name) { - std::string key; - key.reserve(sizeof(PartitionID) + name.size()); - int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - key.append(reinterpret_cast(&item), sizeof(int32_t)) - .append(name.data(), name.size()); - return key; -} - -// static -std::string NebulaKeyUtilsV2::vertexPrefix(size_t vIdLen, - PartitionID partId, - VertexID vId, - TagID tagId) { - CHECK_GE(vIdLen, vId.size()); - tagId &= kTagMaskSet; - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - - std::string key; - key.reserve(sizeof(PartitionID) + vIdLen + sizeof(TagID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(vId.data(), vId.size()) - .append(vIdLen - vId.size(), '\0') - .append(reinterpret_cast(&tagId), sizeof(TagID)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::vertexPrefix(size_t vIdLen, PartitionID partId, VertexID vId) { - CHECK_GE(vIdLen, vId.size()); - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - std::string key; - key.reserve(sizeof(PartitionID) + vIdLen); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(vId.data(), vId.size()) - .append(vIdLen - vId.size(), '\0'); - return key; -} - -// static -std::string NebulaKeyUtilsV2::edgePrefix(size_t vIdLen, - PartitionID partId, - VertexID srcId, - EdgeType type) { - CHECK_GE(vIdLen, srcId.size()); - type |= kEdgeMaskSet; - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - - std::string key; - key.reserve(sizeof(PartitionID) + vIdLen + sizeof(EdgeType)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(srcId.data(), srcId.size()) - .append(vIdLen - srcId.size(), '\0') - .append(reinterpret_cast(&type), sizeof(EdgeType)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::edgePrefix(size_t vIdLen, PartitionID partId, VertexID srcId) { - CHECK_GE(vIdLen, srcId.size()); - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - std::string key; - key.reserve(sizeof(PartitionID) + vIdLen); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(srcId.data(), srcId.size()) - .append(vIdLen - srcId.size(), '\0'); - return key; -} - -// static -std::string NebulaKeyUtilsV2::edgePrefix(size_t vIdLen, - PartitionID partId, - VertexID srcId, - EdgeType type, - EdgeRanking rank, - VertexID dstId) { - CHECK_GE(vIdLen, srcId.size()); - CHECK_GE(vIdLen, dstId.size()); - type |= kEdgeMaskSet; - int32_t item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - std::string key; - key.reserve(sizeof(PartitionID) + (vIdLen << 1) + sizeof(EdgeType) + sizeof(EdgeRanking)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)) - .append(srcId.data(), srcId.size()) - .append(vIdLen - srcId.size(), '\0') - .append(reinterpret_cast(&type), sizeof(EdgeType)) - .append(reinterpret_cast(&rank), sizeof(EdgeRanking)) - .append(dstId.data(), dstId.size()) - .append(vIdLen - dstId.size(), '\0'); - return key; -} - -// static -std::string NebulaKeyUtilsV2::partPrefix(PartitionID partId) { - PartitionID item = (partId << kPartitionOffset) | static_cast(NebulaKeyTypeV2::kData); - std::string key; - key.reserve(sizeof(PartitionID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)); - return key; -} - -// static -std::string NebulaKeyUtilsV2::snapshotPrefix(PartitionID partId) { - // snapshot of meta would be all key-value pairs - if (partId == 0) { - return ""; - } - return partPrefix(partId); -} - -std::string NebulaKeyUtilsV2::systemPrefix() { - int8_t type = static_cast(NebulaKeyTypeV2::kSystem); - std::string key; - key.reserve(sizeof(int8_t)); - key.append(reinterpret_cast(&type), sizeof(int8_t)); - return key; -} - -std::string NebulaKeyUtilsV2::toLockKey(const folly::StringPiece& rawKey, bool enableMvcc) { - EdgeVersion verPlaceHolder = 0; - EdgeVersion ver = 0; - if (enableMvcc) { - auto offset = rawKey.size() - sizeof(EdgeVersion); - ver = readInt(rawKey.data() + offset, sizeof(EdgeVersion)); - } - - auto lockKey = NebulaKeyUtilsV2::keyWithNoVersion(rawKey).str(); - lockKey.append(reinterpret_cast(&verPlaceHolder), sizeof(EdgeVersion)); - lockKey.append(reinterpret_cast(&ver), sizeof(EdgeVersion)); - return lockKey + kLockSuffix; -} - -std::string NebulaKeyUtilsV2::toEdgeKey(const folly::StringPiece& lockKey, bool enableMvcc) { - // edges in toss space must have edge ver greater then 0. - EdgeVersion ver = enableMvcc ? NebulaKeyUtilsV2::getLockVersion(lockKey) : 1; - - auto rawKey = NebulaKeyUtilsV2::lockWithNoVersion(lockKey).str(); - rawKey.append(reinterpret_cast(&ver), sizeof(EdgeVersion)); - return rawKey; -} - -} // namespace nebula diff --git a/src/tools/db-upgrade/NebulaKeyUtilsV2.h b/src/tools/db-upgrade/NebulaKeyUtilsV2.h deleted file mode 100644 index ad35cb5ae26..00000000000 --- a/src/tools/db-upgrade/NebulaKeyUtilsV2.h +++ /dev/null @@ -1,304 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#ifndef TOOLS_DBUPGRADE_NEBULAKEYUTILSV2_H_ -#define TOOLS_DBUPGRADE_NEBULAKEYUTILSV2_H_ - -#include "common/utils/Types.h" - -namespace nebula { - -enum class NebulaKeyTypeV2 : uint32_t { - kData = 0x00000001, - kIndex = 0x00000002, - kUUID = 0x00000003, - kSystem = 0x00000004, - kOperation = 0x00000005, -}; - -/** - * VertexKeyUtils: - * type(1) + partId(3) + vertexId(*) + tagId(4) + version(8) - * - * EdgeKeyUtils: - * type(1) + partId(3) + srcId(*) + edgeType(4) + edgeRank(8) + dstId(*) + - * version(8) - * - * For data in Nebula 1.0, all vertexId is int64_t, so the size would be 8. - * For data in Nebula 2.0, all vertexId is fixed length string according to - * space property. - * - * LockKeyUtils: - * EdgeKeyWithNoVersion + placeHolder(8) + version(8) + suffix(2) - * */ - -const std::string kLockSuffix = "lk"; // NOLINT - -/** - * This class supply some utils for transition between Vertex/Edge and key in - * kvstore for v2.0 rc. - * */ -class NebulaKeyUtilsV2 final { - public: - ~NebulaKeyUtilsV2() = default; - - /* - * Check the validity of vid length - */ - static bool isValidVidLen(size_t vIdLen, VertexID srcvId, VertexID dstvId = ""); - - /** - * Generate tag key for kv store - * */ - static std::string tagKey( - size_t vIdLen, PartitionID partId, VertexID vId, TagID tagId, TagVersion tv); - - /** - * Generate edge key for kv store - * */ - static std::string edgeKey(size_t vIdLen, - PartitionID partId, - VertexID srcId, - EdgeType type, - EdgeRanking rank, - VertexID dstId, - EdgeVersion ev); - - static std::string systemCommitKey(PartitionID partId); - - static std::string systemPartKey(PartitionID partId); - - static std::string kvKey(PartitionID partId, const folly::StringPiece& name); - - /** - * Prefix for vertex - * */ - static std::string vertexPrefix(size_t vIdLen, PartitionID partId, VertexID vId, TagID tagId); - - static std::string vertexPrefix(size_t vIdLen, PartitionID partId, VertexID vId); - - /** - * Prefix for edge - * */ - static std::string edgePrefix(size_t vIdLen, PartitionID partId, VertexID srcId, EdgeType type); - - static std::string edgePrefix(size_t vIdLen, PartitionID partId, VertexID srcId); - - static std::string edgePrefix(size_t vIdLen, - PartitionID partId, - VertexID srcId, - EdgeType type, - EdgeRanking rank, - VertexID dstId); - - static std::string systemPrefix(); - - static std::string partPrefix(PartitionID partId); - - static std::string snapshotPrefix(PartitionID partId); - - static PartitionID getPart(const folly::StringPiece& rawKey) { - return readInt(rawKey.data(), sizeof(PartitionID)) >> 8; - } - - static bool isVertex(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() != kVertexLen + vIdLen) { - return false; - } - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV2)); - auto type = readInt(rawKey.data(), len) & kTypeMask; - if (static_cast(NebulaKeyTypeV2::kData) != type) { - return false; - } - auto offset = sizeof(PartitionID) + vIdLen; - TagID tagId = readInt(rawKey.data() + offset, sizeof(TagID)); - return !(tagId & kTagEdgeMask); - } - - static VertexIDSlice getVertexId(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() != kVertexLen + vIdLen) { - dumpBadKey(rawKey, kVertexLen + vIdLen, vIdLen); - } - auto offset = sizeof(PartitionID); - return rawKey.subpiece(offset, vIdLen); - } - - static TagID getTagId(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() != kVertexLen + vIdLen) { - dumpBadKey(rawKey, kVertexLen + vIdLen, vIdLen); - } - auto offset = sizeof(PartitionID) + vIdLen; - return readInt(rawKey.data() + offset, sizeof(TagID)); - } - - static bool isEdge(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() != kEdgeLen + (vIdLen << 1)) { - return false; - } - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV2)); - auto type = readInt(rawKey.data(), len) & kTypeMask; - if (static_cast(NebulaKeyTypeV2::kData) != type) { - return false; - } - auto offset = sizeof(PartitionID) + vIdLen; - EdgeType etype = readInt(rawKey.data() + offset, sizeof(EdgeType)); - return etype & kTagEdgeMask; - } - - static bool isLock(size_t vIdLen, const folly::StringPiece& rawKey) { - auto len = rawKey.size() - sizeof(EdgeVersion) - kLockSuffix.size(); - return isEdge(vIdLen, folly::StringPiece(rawKey.begin(), len)); - } - - static bool isSystemCommit(const folly::StringPiece& rawKey) { - if (rawKey.size() != kSystemLen) { - return false; - } - auto position = rawKey.data() + sizeof(PartitionID); - auto len = sizeof(NebulaSystemKeyType); - auto type = readInt(position, len); - return static_cast(NebulaSystemKeyType::kSystemCommit) == type; - } - - static bool isSystemPart(const folly::StringPiece& rawKey) { - if (rawKey.size() != kSystemLen) { - return false; - } - auto position = rawKey.data() + sizeof(PartitionID); - auto len = sizeof(NebulaSystemKeyType); - auto type = readInt(position, len); - return static_cast(NebulaSystemKeyType::kSystemPart) == type; - } - - static VertexIDSlice getSrcId(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() < kEdgeLen + (vIdLen << 1)) { - dumpBadKey(rawKey, kEdgeLen + (vIdLen << 1), vIdLen); - } - auto offset = sizeof(PartitionID); - return rawKey.subpiece(offset, vIdLen); - } - - static VertexIDSlice getDstId(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() < kEdgeLen + (vIdLen << 1)) { - dumpBadKey(rawKey, kEdgeLen + (vIdLen << 1), vIdLen); - } - auto offset = sizeof(PartitionID) + vIdLen + sizeof(EdgeType) + sizeof(EdgeRanking); - return rawKey.subpiece(offset, vIdLen); - } - - static EdgeType getEdgeType(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() < kEdgeLen + (vIdLen << 1)) { - dumpBadKey(rawKey, kEdgeLen + (vIdLen << 1), vIdLen); - } - auto offset = sizeof(PartitionID) + vIdLen; - EdgeType type = readInt(rawKey.data() + offset, sizeof(EdgeType)); - return type > 0 ? type & kTagEdgeValueMask : type; - } - - static EdgeRanking getRank(size_t vIdLen, const folly::StringPiece& rawKey) { - if (rawKey.size() < kEdgeLen + (vIdLen << 1)) { - dumpBadKey(rawKey, kEdgeLen + (vIdLen << 1), vIdLen); - } - auto offset = sizeof(PartitionID) + vIdLen + sizeof(EdgeType); - return readInt(rawKey.data() + offset, sizeof(EdgeRanking)); - } - - static int64_t getVersion(size_t vIdLen, const folly::StringPiece& rawKey) { - if (isVertex(vIdLen, rawKey) || isEdge(vIdLen, rawKey)) { - auto offset = rawKey.size() - sizeof(int64_t); - return readInt(rawKey.data() + offset, sizeof(int64_t)); - } else if (isLock(vIdLen, rawKey)) { - return getLockVersion(rawKey); - } else { - LOG(FATAL) << "key is not one of vertex, edge or lock"; - } - return 0; // will not runs here, just for satisfied g++ - } - - static bool isDataKey(const folly::StringPiece& key) { - constexpr int32_t len = static_cast(sizeof(NebulaKeyTypeV2)); - auto type = readInt(key.data(), len) & kTypeMask; - return static_cast(NebulaKeyTypeV2::kData) == type; - } - - static folly::StringPiece keyWithNoVersion(const folly::StringPiece& rawKey) { - // TODO(heng) We should change the method if varint data version supported. - return rawKey.subpiece(0, rawKey.size() - sizeof(int64_t)); - } - - /** - * @brief gen edge key from lock, this will used at resume - * if enableMvcc ver of edge and lock will be same, - * else ver of lock should be 0, and ver of edge should be 1 - */ - static std::string toEdgeKey(const folly::StringPiece& lockKey, bool enableMvcc = false); - - /** - * @brief gen edge lock from lock - * if enableMvcc ver of edge and lock will be same, - * else ver of lock should be 0, and ver of edge should be 1 - */ - static std::string toLockKey(const folly::StringPiece& rawKey, bool enableMvcc = false); - - static EdgeVersion getLockVersion(const folly::StringPiece& rawKey) { - // TODO(liuyu) We should change the method if varint data version - // supported. - auto offset = rawKey.size() - sizeof(int64_t) * 2 - kLockSuffix.size(); - return readInt(rawKey.data() + offset, sizeof(int64_t)); - } - - static folly::StringPiece lockWithNoVersion(const folly::StringPiece& rawKey) { - // TODO(liuyu) We should change the method if varint data version - // supported. - return rawKey.subpiece(0, rawKey.size() - sizeof(int64_t) * 2 - kLockSuffix.size()); - } - - static void dumpBadKey(const folly::StringPiece& rawKey, size_t expect, size_t vIdLen) { - std::stringstream msg; - msg << "rawKey.size() != expect size" - << ", rawKey.size() = " << rawKey.size() << ", expect = " << expect - << ", vIdLen = " << vIdLen << ", rawkey hex format:\n"; - msg << folly::hexDump(rawKey.data(), rawKey.size()); - LOG(FATAL) << msg.str(); - } - - private: - NebulaKeyUtilsV2() = delete; - - private: - // size of vertex key except vertexId - static constexpr int32_t kVertexLen = sizeof(PartitionID) + sizeof(TagID) + sizeof(TagVersion); - - // size of vertex key except srcId and dstId - static constexpr int32_t kEdgeLen = - sizeof(PartitionID) + sizeof(EdgeType) + sizeof(EdgeRanking) + sizeof(EdgeVersion); - - static constexpr int32_t kSystemLen = sizeof(PartitionID) + sizeof(NebulaSystemKeyType); - - // The partition id offset in 4 Bytes - static constexpr uint8_t kPartitionOffset = 8; - - // The key type bits Mask - // See KeyType enum - static constexpr uint32_t kTypeMask = 0x000000FF; - - // The Tag/Edge type bit Mask, the most significant bit is to indicate sign, - // the next bit is to indicate it is tag or edge, 0 for Tag, 1 for Edge - static constexpr uint32_t kTagEdgeMask = 0x40000000; - // For extract Tag/Edge value - static constexpr uint32_t kTagEdgeValueMask = ~kTagEdgeMask; - // Write edge by &= - static constexpr uint32_t kEdgeMaskSet = kTagEdgeMask; - // Write Tag by |= - static constexpr uint32_t kTagMaskSet = ~kTagEdgeMask; - - static constexpr int32_t kVertexIndexLen = sizeof(PartitionID) + sizeof(IndexID); - - static constexpr int32_t kEdgeIndexLen = - sizeof(PartitionID) + sizeof(IndexID) + sizeof(EdgeRanking); -}; - -} // namespace nebula -#endif // TOOLS_DBUPGRADE_NEBULAKEYUTILSV2_H_ diff --git a/src/tools/db-upgrade/NebulaKeyUtilsV3.cpp b/src/tools/db-upgrade/NebulaKeyUtilsV3.cpp deleted file mode 100644 index e7e7033e389..00000000000 --- a/src/tools/db-upgrade/NebulaKeyUtilsV3.cpp +++ /dev/null @@ -1,26 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ - -#include "tools/db-upgrade/NebulaKeyUtilsV3.h" - -namespace nebula { -std::string NebulaKeyUtilsV3::partTagPrefix(PartitionID partId) { - PartitionID item = (partId << kPartitionOffset) | static_cast(kTag_); - std::string key; - key.reserve(sizeof(PartitionID)); - key.append(reinterpret_cast(&item), sizeof(PartitionID)); - return key; -} -std::string NebulaKeyUtilsV3::getVertexKey(folly::StringPiece tagKey) { - std::string key = tagKey.toString(); - key[0] = static_cast(kVertex); - key.resize(key.size() - sizeof(TagID)); - return key; -} -std::string NebulaKeyUtilsV3::dataVersionValue() { - return "3.0"; -} - -} // namespace nebula diff --git a/src/tools/db-upgrade/NebulaKeyUtilsV3.h b/src/tools/db-upgrade/NebulaKeyUtilsV3.h deleted file mode 100644 index b486690067f..00000000000 --- a/src/tools/db-upgrade/NebulaKeyUtilsV3.h +++ /dev/null @@ -1,20 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License. - */ -#ifndef TOOLS_DB_UPGRADE_NEBULAKEYUTILSV3_H -#define TOOLS_DB_UPGRADE_NEBULAKEYUTILSV3_H -#include "common/utils/Types.h" -namespace nebula { -class NebulaKeyUtilsV3 { - public: - static std::string partTagPrefix(PartitionID partId); - static std::string getVertexKey(folly::StringPiece tagKey); - static std::string dataVersionValue(); - - private: - enum NebulaKeyTypeV3 : uint32_t { kTag_ = 0x00000001, kVertex = 0x00000007 }; -}; - -} // namespace nebula -#endif diff --git a/tests/tck/features/delete/DeleteTag.IntVid.feature b/tests/tck/features/delete/DeleteTag.IntVid.feature index aaffae4d825..af279dd4701 100644 --- a/tests/tck/features/delete/DeleteTag.IntVid.feature +++ b/tests/tck/features/delete/DeleteTag.IntVid.feature @@ -41,7 +41,6 @@ Feature: Delete int vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON bachelor hash("Tim Duncan") YIELD bachelor.name, bachelor.speciality @@ -95,14 +94,12 @@ Feature: Delete int vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON bachelor hash("Tim Duncan") YIELD bachelor.name, bachelor.speciality """ Then the result should be, in any order: | bachelor.name | bachelor.speciality | - | EMPTY | EMPTY | When executing query: """ LOOKUP ON player WHERE player.name == "Tim Duncan" YIELD id(vertex) as id @@ -149,14 +146,12 @@ Feature: Delete int vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON bachelor hash("Tim Duncan") YIELD bachelor.name, bachelor.speciality """ Then the result should be, in any order: | bachelor.name | bachelor.speciality | - | EMPTY | EMPTY | When executing query: """ LOOKUP ON player WHERE player.name == "Tim Duncan" YIELD id(vertex) as id @@ -210,14 +205,12 @@ Feature: Delete int vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON player hash("Tony Parker") YIELD player.name, player.age """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ LOOKUP ON player WHERE player.name == "Tim Duncan" YIELD id(vertex) as id @@ -263,7 +256,6 @@ Feature: Delete int vid of tag """ Then the result should be, in any order: | team.name | - | EMPTY | # delete tag from pipe and normal When executing query: """ @@ -303,7 +295,6 @@ Feature: Delete int vid of tag """ Then the result should be, in any order: | team.name | - | EMPTY | # delete one tag from var and normal When executing query: """ diff --git a/tests/tck/features/delete/DeleteTag.feature b/tests/tck/features/delete/DeleteTag.feature index 5770bdcd133..4e01b0cdeff 100644 --- a/tests/tck/features/delete/DeleteTag.feature +++ b/tests/tck/features/delete/DeleteTag.feature @@ -41,7 +41,6 @@ Feature: Delete string vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON bachelor "Tim Duncan" YIELD bachelor.name, bachelor.speciality @@ -95,14 +94,12 @@ Feature: Delete string vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON bachelor "Tim Duncan" YIELD bachelor.name, bachelor.speciality """ Then the result should be, in any order: | bachelor.name | bachelor.speciality | - | EMPTY | EMPTY | When executing query: """ LOOKUP ON player WHERE player.name == "Tim Duncan" YIELD id(vertex) as id @@ -149,14 +146,12 @@ Feature: Delete string vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON bachelor "Tim Duncan" YIELD bachelor.name, bachelor.speciality """ Then the result should be, in any order: | bachelor.name | bachelor.speciality | - | EMPTY | EMPTY | When executing query: """ LOOKUP ON player WHERE player.name == "Tim Duncan" YIELD id(vertex) as id @@ -210,14 +205,12 @@ Feature: Delete string vid of tag """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON player "Tony Parker" YIELD player.name, player.age """ Then the result should be, in any order: | player.name | player.age | - | EMPTY | EMPTY | When executing query: """ LOOKUP ON player WHERE player.name == "Tim Duncan" YIELD id(vertex) as id @@ -263,7 +256,6 @@ Feature: Delete string vid of tag """ Then the result should be, in any order: | team.name | - | EMPTY | # delete tag from pipe and normal When executing query: """ @@ -303,7 +295,6 @@ Feature: Delete string vid of tag """ Then the result should be, in any order: | team.name | - | EMPTY | # delete one tag from var and normal When executing query: """ diff --git a/tests/tck/features/go/GO.IntVid.feature b/tests/tck/features/go/GO.IntVid.feature index 757b31efc45..27eb032b59d 100644 --- a/tests/tck/features/go/GO.IntVid.feature +++ b/tests/tck/features/go/GO.IntVid.feature @@ -1383,14 +1383,14 @@ Feature: IntegerVid Go Sentence """ Then the result should be, in any order, with relax comparison, and the columns 0,1 should be hashed: | serve._dst | like._dst | serve.start_year | like.likeness | $$.player.name | - | "Thunders" | EMPTY | 2008 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2008 | EMPTY | NULL | | EMPTY | "Paul George" | EMPTY | 90 | "Paul George" | | EMPTY | "James Harden" | EMPTY | 90 | "James Harden" | - | "Pacers" | EMPTY | 2010 | EMPTY | EMPTY | - | "Thunders" | EMPTY | 2017 | EMPTY | EMPTY | + | "Pacers" | EMPTY | 2010 | EMPTY | NULL | + | "Thunders" | EMPTY | 2017 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 95 | "Russell Westbrook" | - | "Thunders" | EMPTY | 2009 | EMPTY | EMPTY | - | "Rockets" | EMPTY | 2012 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2009 | EMPTY | NULL | + | "Rockets" | EMPTY | 2012 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 80 | "Russell Westbrook" | When executing query: """ @@ -1399,14 +1399,14 @@ Feature: IntegerVid Go Sentence """ Then the result should be, in any order, with relax comparison, and the columns 0,1 should be hashed: | serve._dst | like._dst | serve.start_year | like.likeness | $$.player.name | - | "Thunders" | EMPTY | 2008 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2008 | EMPTY | NULL | | EMPTY | "Paul George" | EMPTY | 90 | "Paul George" | | EMPTY | "James Harden" | EMPTY | 90 | "James Harden" | - | "Pacers" | EMPTY | 2010 | EMPTY | EMPTY | - | "Thunders" | EMPTY | 2017 | EMPTY | EMPTY | + | "Pacers" | EMPTY | 2010 | EMPTY | NULL | + | "Thunders" | EMPTY | 2017 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 95 | "Russell Westbrook" | - | "Thunders" | EMPTY | 2009 | EMPTY | EMPTY | - | "Rockets" | EMPTY | 2012 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2009 | EMPTY | NULL | + | "Rockets" | EMPTY | 2012 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 80 | "Russell Westbrook" | When executing query: """ @@ -1480,8 +1480,8 @@ Feature: IntegerVid Go Sentence GO FROM hash('Tim Duncan') OVER serve YIELD $$.player.name as name """ Then the result should be, in any order, with relax comparison: - | name | - | EMPTY | + | name | + | NULL | Scenario: Integer Vid zero step When executing query: diff --git a/tests/tck/features/go/GO.feature b/tests/tck/features/go/GO.feature index c76057d5bbe..2132813b73c 100644 --- a/tests/tck/features/go/GO.feature +++ b/tests/tck/features/go/GO.feature @@ -339,7 +339,7 @@ Feature: Go Sentence When executing query: """ GO FROM "Paul Gasol" OVER * - WHERE $$.player.name IS NOT EMPTY + WHERE $$.player.name IS NOT NULL YIELD like._dst """ Then the result should be, in any order, with relax comparison: @@ -349,7 +349,7 @@ Feature: Go Sentence When executing query: """ GO FROM "Paul Gasol" OVER * - WHERE $$.player.name IS EMPTY + WHERE $$.player.name IS NULL YIELD like._dst """ Then the result should be, in any order, with relax comparison: @@ -1467,14 +1467,14 @@ Feature: Go Sentence """ Then the result should be, in any order, with relax comparison: | serve._dst | like._dst | serve.start_year | like.likeness | $$.player.name | - | "Thunders" | EMPTY | 2008 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2008 | EMPTY | NULL | | EMPTY | "Paul George" | EMPTY | 90 | "Paul George" | | EMPTY | "James Harden" | EMPTY | 90 | "James Harden" | - | "Pacers" | EMPTY | 2010 | EMPTY | EMPTY | - | "Thunders" | EMPTY | 2017 | EMPTY | EMPTY | + | "Pacers" | EMPTY | 2010 | EMPTY | NULL | + | "Thunders" | EMPTY | 2017 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 95 | "Russell Westbrook" | - | "Thunders" | EMPTY | 2009 | EMPTY | EMPTY | - | "Rockets" | EMPTY | 2012 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2009 | EMPTY | NULL | + | "Rockets" | EMPTY | 2012 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 80 | "Russell Westbrook" | When executing query: """ @@ -1483,14 +1483,14 @@ Feature: Go Sentence """ Then the result should be, in any order, with relax comparison: | serve._dst | like._dst | serve.start_year | like.likeness | $$.player.name | - | "Thunders" | EMPTY | 2008 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2008 | EMPTY | NULL | | EMPTY | "Paul George" | EMPTY | 90 | "Paul George" | | EMPTY | "James Harden" | EMPTY | 90 | "James Harden" | - | "Pacers" | EMPTY | 2010 | EMPTY | EMPTY | - | "Thunders" | EMPTY | 2017 | EMPTY | EMPTY | + | "Pacers" | EMPTY | 2010 | EMPTY | NULL | + | "Thunders" | EMPTY | 2017 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 95 | "Russell Westbrook" | - | "Thunders" | EMPTY | 2009 | EMPTY | EMPTY | - | "Rockets" | EMPTY | 2012 | EMPTY | EMPTY | + | "Thunders" | EMPTY | 2009 | EMPTY | NULL | + | "Rockets" | EMPTY | 2012 | EMPTY | NULL | | EMPTY | "Russell Westbrook" | EMPTY | 80 | "Russell Westbrook" | When executing query: """ @@ -1564,8 +1564,8 @@ Feature: Go Sentence GO FROM 'Tim Duncan' OVER serve YIELD $$.player.name as name """ Then the result should be, in any order, with relax comparison: - | name | - | EMPTY | + | name | + | NULL | Scenario: zero step When executing query: diff --git a/tests/tck/features/go/GoYieldVertexEdge.feature b/tests/tck/features/go/GoYieldVertexEdge.feature index 5fbba5aec71..e32b5b96031 100644 --- a/tests/tck/features/go/GoYieldVertexEdge.feature +++ b/tests/tck/features/go/GoYieldVertexEdge.feature @@ -345,7 +345,7 @@ Feature: Go Yield Vertex And Edge Sentence When executing query: """ GO FROM "Paul Gasol" OVER * - WHERE $$.player.name IS NOT EMPTY + WHERE $$.player.name IS NOT NULL YIELD edge as e """ Then the result should be, in any order, with relax comparison: @@ -355,7 +355,7 @@ Feature: Go Yield Vertex And Edge Sentence When executing query: """ GO FROM "Paul Gasol" OVER * - WHERE $$.player.name IS EMPTY + WHERE $$.player.name IS NULL YIELD type(edge) as type """ Then the result should be, in any order, with relax comparison: @@ -1383,13 +1383,13 @@ Feature: Go Yield Vertex And Edge Sentence | dst | serve.start_year | like.likeness | $$.player.name | | "James Harden" | EMPTY | 90 | "James Harden" | | "Paul George" | EMPTY | 90 | "Paul George" | - | "Thunders" | 2008 | EMPTY | EMPTY | + | "Thunders" | 2008 | EMPTY | NULL | | "Russell Westbrook" | EMPTY | 80 | "Russell Westbrook" | - | "Rockets" | 2012 | EMPTY | EMPTY | - | "Thunders" | 2009 | EMPTY | EMPTY | + | "Rockets" | 2012 | EMPTY | NULL | + | "Thunders" | 2009 | EMPTY | NULL | | "Russell Westbrook" | EMPTY | 95 | "Russell Westbrook" | - | "Pacers" | 2010 | EMPTY | EMPTY | - | "Thunders" | 2017 | EMPTY | EMPTY | + | "Pacers" | 2010 | EMPTY | NULL | + | "Thunders" | 2017 | EMPTY | NULL | When executing query: """ GO 1 TO 2 STEPS FROM 'Russell Westbrook' OVER * REVERSELY YIELD edge as e diff --git a/tests/tck/features/insert/insertVertexOnly.feature b/tests/tck/features/insert/insertVertexOnly.feature index f127d84a81a..9419e29e54b 100644 --- a/tests/tck/features/insert/insertVertexOnly.feature +++ b/tests/tck/features/insert/insertVertexOnly.feature @@ -19,6 +19,10 @@ Feature: insert vertex without tag When executing query and retrying it on failure every 6 seconds for 3 times: """ INSERT VERTEX VALUES 1:(),2:(),3:(); + """ + Then a SemanticError should be raised at runtime: Insert vertex is forbidden, please speicify the tag + When executing query: + """ INSERT EDGE e() VALUES 1->2:(),2->3:(); """ Then the execution should be successful @@ -34,7 +38,5 @@ Feature: insert vertex without tag FETCH PROP ON * 1,2 yield vertex AS v; """ Then the result should be, in any order, with relax comparison: - | v | - | (1) | - | (2) | + | v | Then drop the used space diff --git a/tests/tck/features/lookup/LookUp.feature b/tests/tck/features/lookup/LookUp.feature index 99c5624e8d2..f6be6fc856f 100644 --- a/tests/tck/features/lookup/LookUp.feature +++ b/tests/tck/features/lookup/LookUp.feature @@ -937,6 +937,33 @@ Feature: LookUpTest_Vid_String | '105' | | '106' | | '121' | + When executing query: + """ + LOOKUP ON player where player.age < 100 YIELD player.age as age + """ + Then the result should be, in any order: + | age | + | 42 | + | 36 | + | 33 | + | 35 | + | 28 | + | 21 | + | 21 | + | 20 | + When executing query: + """ + LOOKUP ON player where player.age < 100 YIELD distinct player.age as age + """ + Then the result should be, in any order: + | age | + | 42 | + | 36 | + | 33 | + | 35 | + | 28 | + | 21 | + | 20 | When executing query: """ LOOKUP ON player where player.name == "Useless" YIELD id(vertex) as id diff --git a/tests/tck/features/match/MatchById.IntVid.feature b/tests/tck/features/match/MatchById.IntVid.feature index 1bf44590933..bf35ec30285 100644 --- a/tests/tck/features/match/MatchById.IntVid.feature +++ b/tests/tck/features/match/MatchById.IntVid.feature @@ -101,7 +101,6 @@ Feature: Integer Vid Match By Id """ Then the result should be, in any order, with relax comparison: | Type | Name | - | 'like' | NULL | | 'serve' | 'Cavaliers' | | 'serve' | 'Heat' | | 'serve' | 'Cavaliers' | @@ -114,7 +113,6 @@ Feature: Integer Vid Match By Id """ Then the result should be, in any order, with relax comparison: | Type | Name | - | 'like' | NULL | | 'serve' | 'Cavaliers' | | 'serve' | 'Heat' | | 'serve' | 'Cavaliers' | diff --git a/tests/tck/features/match/SeekByEdge.feature b/tests/tck/features/match/SeekByEdge.feature index a333e520d94..dfdb3a50f4f 100644 --- a/tests/tck/features/match/SeekByEdge.feature +++ b/tests/tck/features/match/SeekByEdge.feature @@ -333,158 +333,6 @@ Feature: Match seek by edge """ Then the result should be, in any order: | player.player.name | team.team.name | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | - | NULL | NULL | | "Vince Carter" | "Nets" | | "Jason Kidd" | "Nets" | | "Grant Hill" | "Pistons" | @@ -1291,6 +1139,7 @@ Feature: Match seek by edge Then the result should be, in any order: | p1.player.name | p2.team.name | + @wtf Scenario Outline: Seek by edge with properties When executing query: """ @@ -1316,8 +1165,6 @@ Feature: Match seek by edge | player | team | | "Paul Gasol" | "Grizzlies" | | "Jason Kidd" | "Nets" | - | NULL | NULL | - | NULL | NULL | When executing query: """ match (player)-[s:serve]->(team) where s.start_year == 2001 return player.player.name AS player, team.team.name AS team @@ -1342,8 +1189,6 @@ Feature: Match seek by edge | player | team | | "Paul Gasol" | "Grizzlies" | | "Jason Kidd" | "Nets" | - | NULL | NULL | - | NULL | NULL | Scenario Outline: Seek by edge with range with properties When executing query: diff --git a/tests/tck/features/match/With.feature b/tests/tck/features/match/With.feature index d4cdca681c0..022dbd4965c 100644 --- a/tests/tck/features/match/With.feature +++ b/tests/tck/features/match/With.feature @@ -133,8 +133,8 @@ Feature: With clause RETURN collect(names) """ Then the result should be, in any order, with relax comparison: - | collect(names) | - | ["Tony Parker", "Tiago Splitter", "Shaquille O'Neal", "Marco Belinelli"] | + | collect(names) | + | ["Tony Parker", "Tiago Splitter", "Shaquille O'Neal", "Marco Belinelli", "Manu Ginobili"] | When profiling query: """ MATCH (v:player) diff --git a/tests/tck/features/optimizer/PrunePropertiesRule.feature b/tests/tck/features/optimizer/PrunePropertiesRule.feature index eaf97df4b5d..d7452f0afc3 100644 --- a/tests/tck/features/optimizer/PrunePropertiesRule.feature +++ b/tests/tck/features/optimizer/PrunePropertiesRule.feature @@ -36,12 +36,12 @@ Feature: Prune Properties rule | "Tony Parker" | | "Tony Parker" | And the execution plan should be: - | id | name | dependencies | operator info | - | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 9, \"props\": [\"_tag\"]}]"} | - | 3 | Traverse | 7 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | - | 7 | IndexScan | 2 | | - | 2 | Start | | | + | id | name | dependencies | operator info | + | 8 | Project | 4 | | + | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 8, \"props\": [\"_tag\"]}, {\"tagId\": 9, \"props\": [\"_tag\"]}, {\"tagId\": 10, \"props\": [\"_tag\"]}]"} | + | 3 | Traverse | 7 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | + | 7 | IndexScan | 2 | | + | 2 | Start | | | When profiling query: """ MATCH p = (v:player{name: "Tony Parker"})-[e:like]-(v2) @@ -60,7 +60,7 @@ Feature: Prune Properties rule And the execution plan should be: | id | name | dependencies | operator info | | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 9, \"props\": [\"_tag\"]} ]" } | + | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 8, \"props\": [\"_tag\"]}, {\"tagId\": 9, \"props\": [\"_tag\"]}, {\"tagId\": 10, \"props\": [\"_tag\"]}]" } | | 3 | Traverse | 7 | { "vertexProps": "[{\"props\": [\"name\", \"age\", \"_tag\"], \"tagId\": 9}, {\"props\": [\"name\", \"speciality\", \"_tag\"], \"tagId\": 8}, {\"tagId\": 10, \"props\": [\"name\", \"_tag\"]}]", "edgeProps": "[{\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}]" } | | 7 | IndexScan | 2 | | | 2 | Start | | | @@ -108,12 +108,12 @@ Feature: Prune Properties rule | "like" | | "like" | And the execution plan should be: - | id | name | dependencies | operator info | - | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9} ]" } | - | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | - | 7 | IndexScan | 2 | | - | 2 | Start | | | + | id | name | dependencies | operator info | + | 8 | Project | 4 | | + | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":8}, {\"props\":[\"_tag\"],\"tagId\":9}, {\"props\":[\"_tag\"],\"tagId\":10} ]" } | + | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | + | 7 | IndexScan | 2 | | + | 2 | Start | | | When executing query: """ MATCH (v:player{name: "Tony Parker"})-[:like]-(v2)--(v3) @@ -377,18 +377,18 @@ Feature: Prune Properties rule | scount | | 270 | And the execution plan should be: - | id | name | dependencies | operator info | - | 12 | Aggregate | 13 | | - | 13 | BiInnerJoin | 15, 11 | | - | 15 | Project | 4 | | - | 4 | Traverse | 3 | { "vertexProps": "" } | - | 3 | Traverse | 14 | { "vertexProps": "" } | - | 14 | IndexScan | 2 | | - | 2 | Start | | | - | 11 | Project | 10 | | - | 10 | AppendVertices | 9 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | - | 9 | Traverse | 8 | { "vertexProps": "" } | - | 8 | Argument | | | + | id | name | dependencies | operator info | + | 12 | Aggregate | 13 | | + | 13 | BiInnerJoin | 15, 11 | | + | 15 | Project | 4 | | + | 4 | Traverse | 3 | { "vertexProps": "" } | + | 3 | Traverse | 14 | { "vertexProps": "" } | + | 14 | IndexScan | 2 | | + | 2 | Start | | | + | 11 | Project | 10 | | + | 10 | AppendVertices | 9 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":8}, {\"props\":[\"_tag\"],\"tagId\":9}, {\"props\":[\"_tag\"],\"tagId\":10}]" } | + | 9 | Traverse | 8 | { "vertexProps": "" } | + | 8 | Argument | | | @distonly Scenario: return function @@ -421,14 +421,14 @@ Feature: Prune Properties rule | count(v2) | | 24 | And the execution plan should be: - | id | name | dependencies | operator info | - | 7 | Aggregate | 6 | | - | 6 | Project | 5 | | - | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | - | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | + | id | name | dependencies | operator info | + | 7 | Aggregate | 6 | | + | 6 | Project | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":8}, {\"props\":[\"_tag\"],\"tagId\":9}, {\"props\":[\"_tag\"],\"tagId\":10}]" } | + | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | When profiling query: """ MATCH p = (v1)-[e:like*1..5]->(v2) @@ -439,14 +439,14 @@ Feature: Prune Properties rule | length(p) | | 1 | And the execution plan should be: - | id | name | dependencies | operator info | - | 13 | Project | 11 | | - | 11 | Limit | 5 | | - | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | - | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | + | id | name | dependencies | operator info | + | 13 | Project | 11 | | + | 11 | Limit | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":8}, {\"props\":[\"_tag\"],\"tagId\":9}, {\"props\":[\"_tag\"],\"tagId\":10}]" } | + | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | When profiling query: """ MATCH p = (a:player)-[e:like*1..3]->(b:player{age:39}) @@ -505,12 +505,12 @@ Feature: Prune Properties rule | 14 | Dedup | 13 | | | 13 | Union | 18, 19 | | | 18 | Project | 4 | | - | 4 | AppendVertices | 20 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 4 | AppendVertices | 20 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":8}, {\"props\":[\"_tag\"],\"tagId\":9}, {\"props\":[\"_tag\"],\"tagId\":10}]" } | | 20 | Traverse | 16 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | | 16 | IndexScan | 2 | | | 2 | Start | | | | 19 | Project | 10 | | - | 10 | AppendVertices | 21 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 10 | AppendVertices | 21 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":8}, {\"props\":[\"_tag\"],\"tagId\":9}, {\"props\":[\"_tag\"],\"tagId\":10}]" } | | 21 | Traverse | 17 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}, {\"type\": -3, \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | | 17 | IndexScan | 8 | | | 8 | Start | | | diff --git a/tests/tck/features/ttl/TTL.feature b/tests/tck/features/ttl/TTL.feature index f3f74c6c618..1cb456d209d 100644 --- a/tests/tck/features/ttl/TTL.feature +++ b/tests/tck/features/ttl/TTL.feature @@ -393,36 +393,31 @@ Feature: TTLTest FETCH PROP ON person "1" YIELD vertex as node; """ Then the result should be, in any order, with relax comparison: - | node | - | ("1") | + | node | When executing query: """ FETCH PROP ON person "1" YIELD person.id as id """ Then the result should be, in any order: - | id | - | EMPTY | + | id | When executing query: """ FETCH PROP ON * "1" YIELD person.id, career.id """ Then the result should be, in any order: | person.id | career.id | - | EMPTY | EMPTY | When executing query: """ FETCH PROP ON person "2" YIELD person.id """ Then the result should be, in any order: | person.id | - | EMPTY | When executing query: """ FETCH PROP ON person "2" YIELD person.id as id """ Then the result should be, in any order: - | id | - | EMPTY | + | id | When executing query: """ FETCH PROP ON career "2" YIELD career.id; @@ -491,6 +486,5 @@ Feature: TTLTest FETCH PROP ON person "1" YIELD person.age as age; """ Then the result should be, in any order: - | age | - | EMPTY | + | age | And drop the used space diff --git a/tests/tck/features/yield/parameter.feature b/tests/tck/features/yield/parameter.feature index fd5ea74fdbf..58bd96d71c9 100644 --- a/tests/tck/features/yield/parameter.feature +++ b/tests/tck/features/yield/parameter.feature @@ -98,6 +98,19 @@ Feature: Parameter | "Tim Duncan" | Scenario: ngql with parameters + When executing query: + """ + LOOKUP ON player where player.age>$p1+40 YIELD player.name AS name + """ + Then the result should be, in any order: + | name | + | "Grant Hill" | + | "Jason Kidd" | + | "Vince Carter" | + | "Tim Duncan" | + | "Shaquille O'Neal" | + | "Steve Nash" | + | "Ray Allen" | When executing query: """ $p1=GO FROM "Tim Duncan" OVER like WHERE like.likeness>$p1 yield like._dst as dst;