Skip to content

Commit

Permalink
Rollback vertex key (#4629)
Browse files Browse the repository at this point in the history
* remove upgrade tool

* rollback

* fetch/go/insert tck

* match tck

* Update GoYieldVertexEdge.feature

* Update GoYieldVertexEdge.feature

* Update GO.feature

* change  use_vertex_key to graph_use_vertex_key

* Update GraphFlags.h

* Update GraphFlags.cpp

Co-authored-by: hs.zhang <22708345+cangfengzhs@users.noreply.github.com>
Co-authored-by: jimingquan <mingquan.ji@vesoft.com>
  • Loading branch information
3 people authored Sep 21, 2022
1 parent 7d5e93f commit 545137a
Show file tree
Hide file tree
Showing 34 changed files with 81 additions and 2,895 deletions.
2 changes: 1 addition & 1 deletion docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Following docker images will be ready in production.
- [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
- [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
- [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader
- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump and meta_dump
1 change: 0 additions & 1 deletion src/common/utils/NebulaKeyUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,6 @@ std::vector<std::string> NebulaKeyUtils::snapshotPrefix(PartitionID partId) {
if (partId == 0) {
result.emplace_back("");
} else {
result.emplace_back(vertexPrefix(partId));
result.emplace_back(tagPrefix(partId));
result.emplace_back(edgePrefix(partId));
result.emplace_back(IndexKeyUtils::indexPrefix(partId));
Expand Down
6 changes: 6 additions & 0 deletions src/common/utils/NebulaKeyUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,12 @@ class NebulaKeyUtils final {
return static_cast<NebulaKeyType>(type) == NebulaKeyType::kEdge;
}

static bool isVertex(const folly::StringPiece& rawKey) {
constexpr int32_t len = static_cast<int32_t>(sizeof(NebulaKeyType));
auto type = readInt<uint32_t>(rawKey.data(), len) & kTypeMask;
return static_cast<NebulaKeyType>(type) == NebulaKeyType::kVertex;
}

static bool isLock(size_t vIdLen, const folly::StringPiece& rawKey) {
return isEdge(vIdLen, rawKey, kLockVersion);
}
Expand Down
2 changes: 2 additions & 0 deletions src/graph/service/GraphFlags.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,3 +103,5 @@ DEFINE_uint32(
gc_worker_size,
0,
"Background garbage clean workers, default number is 0 which means using hardware core size.");

DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key");
3 changes: 3 additions & 0 deletions src/graph/service/GraphFlags.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,7 @@ DECLARE_int32(max_job_size);

DECLARE_bool(enable_async_gc);
DECLARE_uint32(gc_worker_size);

DECLARE_bool(graph_use_vertex_key);

#endif // GRAPH_GRAPHFLAGS_H_
3 changes: 3 additions & 0 deletions src/graph/validator/MutateValidator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ Status InsertVerticesValidator::check() {
}

auto tagItems = sentence->tagItems();
if (!FLAGS_graph_use_vertex_key && tagItems.empty()) {
return Status::SemanticError("Insert vertex is forbidden, please speicify the tag");
}

schemas_.reserve(tagItems.size());

Expand Down
3 changes: 3 additions & 0 deletions src/storage/CompactionFilter.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "common/utils/OperationKeyUtils.h"
#include "kvstore/CompactionFilter.h"
#include "storage/CommonUtils.h"
#include "storage/StorageFlags.h"

namespace nebula {
namespace storage {
Expand All @@ -36,6 +37,8 @@ class StorageCompactionFilter final : public kvstore::KVFilter {
return !edgeValid(spaceId, key, val);
} else if (IndexKeyUtils::isIndexKey(key)) {
return !indexValid(spaceId, key, val);
} else if (!FLAGS_use_vertex_key && NebulaKeyUtils::isVertex(key)) {
return true;
} else if (NebulaKeyUtils::isLock(vIdLen_, key)) {
return !lockValid(spaceId, key);
} else {
Expand Down
2 changes: 2 additions & 0 deletions src/storage/StorageFlags.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,5 @@ DEFINE_bool(query_concurrently,
false,
"whether to run query of each part concurrently, only lookup and "
"go are supported");

DEFINE_bool(use_vertex_key, false, "whether allow insert or query the vertex key");
2 changes: 2 additions & 0 deletions src/storage/StorageFlags.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,6 @@ DECLARE_int32(max_edge_returned_per_vertex);

DECLARE_bool(query_concurrently);

DECLARE_bool(use_vertex_key);

#endif // STORAGE_STORAGEFLAGS_H_
13 changes: 1 addition & 12 deletions src/storage/admin/StatsTask.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
std::unique_ptr<kvstore::KVIterator> tagIter;
auto edgePrefix = NebulaKeyUtils::edgePrefix(part);
std::unique_ptr<kvstore::KVIterator> edgeIter;
auto vertexPrefix = NebulaKeyUtils::vertexPrefix(part);
std::unique_ptr<kvstore::KVIterator> vertexIter;

// When the storage occurs leader change, continue to read data from the
// follower instead of reporting an error.
Expand All @@ -126,11 +124,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
LOG(INFO) << "Stats task failed";
return ret;
}
ret = env_->kvstore_->prefix(spaceId, part, vertexPrefix, &vertexIter, true);
if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
LOG(INFO) << "Stats task failed";
return ret;
}
std::unordered_map<TagID, int64_t> tagsVertices;
std::unordered_map<EdgeType, int64_t> edgetypeEdges;
std::unordered_map<PartitionID, int64_t> positiveRelevancy;
Expand Down Expand Up @@ -173,6 +166,7 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
tagIter->next();
continue;
}
spaceVertices++;
tagsVertices[tagId] += 1;
tagIter->next();
sleepIfScannedSomeRecord(++countToSleep);
Expand Down Expand Up @@ -227,11 +221,6 @@ nebula::cpp2::ErrorCode StatsTask::genSubTask(GraphSpaceID spaceId,
edgeIter->next();
sleepIfScannedSomeRecord(++countToSleep);
}
while (vertexIter && vertexIter->valid()) {
spaceVertices++;
vertexIter->next();
sleepIfScannedSomeRecord(++countToSleep);
}
nebula::meta::cpp2::StatsItem statsItem;

// convert tagId/edgeType to tagName/edgeName
Expand Down
21 changes: 13 additions & 8 deletions src/storage/exec/GetPropNode.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,23 @@ class GetTagPropNode : public QueryNode<VertexID> {
return ret;
}

// if none of the tag node and vertex valid, do not emplace the row
// If none of the tag node valid, will check vertex key if use_vertex_key is true,
// do not emplace the row if the flag is false
if (!std::any_of(tagNodes_.begin(), tagNodes_.end(), [](const auto& tagNode) {
return tagNode->valid();
})) {
auto kvstore = context_->env()->kvstore_;
auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId);
std::string value;
ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value);
if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
if (FLAGS_use_vertex_key) {
auto kvstore = context_->env()->kvstore_;
auto vertexKey = NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId);
std::string value;
ret = kvstore->get(context_->spaceId(), partId, vertexKey, &value);
if (ret == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) {
return nebula::cpp2::ErrorCode::SUCCEEDED;
} else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
return ret;
}
} else {
return nebula::cpp2::ErrorCode::SUCCEEDED;
} else if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
return ret;
}
}

Expand Down
1 change: 0 additions & 1 deletion src/storage/exec/UpdateNode.h
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,6 @@ class UpdateTagNode : public UpdateNode<VertexID> {
}
}
// step 3, insert new vertex data
batchHolder->put(NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId), "");
batchHolder->put(std::move(key_), std::move(nVal));
return encodeBatchValue(batchHolder->getBatch());
}
Expand Down
2 changes: 0 additions & 2 deletions src/storage/mutate/AddVerticesProcessor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
code = nebula::cpp2::ErrorCode::E_INVALID_VID;
break;
}
data.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid), "");
for (auto& newTag : newTags) {
auto tagId = newTag.get_tag_id();
VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId;
Expand Down Expand Up @@ -160,7 +159,6 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re
break;
}

verticeData.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid));
for (const auto& newTag : newTags) {
auto tagId = newTag.get_tag_id();
VLOG(3) << "PartitionID: " << partId << ", VertexID: " << vid << ", TagID: " << tagId;
Expand Down
1 change: 0 additions & 1 deletion src/tools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,3 @@ nebula_add_subdirectory(simple-kv-verify)
endif()
nebula_add_subdirectory(meta-dump)
nebula_add_subdirectory(db-dump)
nebula_add_subdirectory(db-upgrade)
29 changes: 0 additions & 29 deletions src/tools/db-upgrade/CMakeLists.txt

This file was deleted.

Loading

0 comments on commit 545137a

Please sign in to comment.