From fbb935637046456e1a0cf5a1b0b124c1625827b2 Mon Sep 17 00:00:00 2001 From: jimingquan Date: Tue, 3 Jan 2023 16:20:53 +0800 Subject: [PATCH 01/51] optimize match node label (#5176) --- src/graph/executor/query/TraverseExecutor.cpp | 68 ++++++++----------- src/graph/executor/query/TraverseExecutor.h | 6 +- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp index f0e319c9462..f6cf7910acf 100644 --- a/src/graph/executor/query/TraverseExecutor.cpp +++ b/src/graph/executor/query/TraverseExecutor.cpp @@ -36,7 +36,6 @@ Status TraverseExecutor::buildRequestVids() { auto iter = static_cast(inputIter); size_t iterSize = iter->size(); vids_.reserve(iterSize); - initVids_.reserve(iterSize); auto* src = traverse_->src(); QueryExpressionContext ctx(ectx_); @@ -56,11 +55,9 @@ Status TraverseExecutor::buildRequestVids() { } if (uniqueVid.emplace(vid).second) { vids_.emplace_back(vid); - initVids_.emplace_back(vid); } } } else { - initVids_.reserve(iterSize); const auto& spaceInfo = qctx()->rctx()->session()->space(); const auto& metaVidType = *(spaceInfo.spaceDesc.vid_type_ref()); auto vidType = SchemaUtil::propTypeToValueType(metaVidType.get_type()); @@ -71,7 +68,6 @@ Status TraverseExecutor::buildRequestVids() { continue; } vids_.emplace_back(vid); - initVids_.emplace_back(vid); } } return Status::OK(); @@ -164,27 +160,18 @@ folly::Future TraverseExecutor::handleResponse(RpcResponse&& resps) { auto listVal = std::make_shared(std::move(list)); auto iter = std::make_unique(listVal); if (currentStep_ == 1) { - if (range_ && range_->min() == 0) { - result_.rows = buildZeroStepPath(iter.get()); - } - // match (v)-[e:Rel]-(v1:Label1)-[e1*2]->() where id(v0) in [6, 23] return v1 - // the attributes of v1 will be obtained in the second traverse operator - // If the conditions are not met, the path in the previous step needs to be filtered out - std::unordered_set existVids; - existVids.reserve(iter->numRows()); + initVertices_.reserve(iter->numRows()); auto vertices = iter->getVertices(); + // match (v)-[e:Rel]-(v1:Label1)-[e1*2]->() where id(v0) in [6, 23] return v1 + // save the vertex that meets the filter conditions as the starting vertex of the current + // traverse for (auto& vertex : vertices.values) { if (vertex.isVertex()) { - existVids.emplace(vertex); + initVertices_.emplace_back(vertex); } } - auto initVidIter = initVids_.begin(); - while (initVidIter != initVids_.end()) { - if (existVids.find(*initVidIter) == existVids.end()) { - initVidIter = initVids_.erase(initVidIter); - } else { - initVidIter++; - } + if (range_ && range_->min() == 0) { + result_.rows = buildZeroStepPath(); } } expand(iter.get()); @@ -248,15 +235,14 @@ void TraverseExecutor::expand(GetNeighborsIter* iter) { } } -std::vector TraverseExecutor::buildZeroStepPath(GetNeighborsIter* iter) { - if (!iter || iter->numRows() == 0) { +std::vector TraverseExecutor::buildZeroStepPath() { + if (initVertices_.empty()) { return std::vector(); } std::vector result; - result.reserve(iter->size()); - auto vertices = iter->getVertices(); + result.reserve(initVertices_.size()); if (traverse_->trackPrevPath()) { - for (auto& vertex : vertices.values) { + for (auto& vertex : initVertices_) { auto dstIter = dst2PathsMap_.find(vertex); if (dstIter == dst2PathsMap_.end()) { continue; @@ -272,7 +258,7 @@ std::vector TraverseExecutor::buildZeroStepPath(GetNeighborsIter* iter) { } } } else { - for (auto& vertex : vertices.values) { + for (auto& vertex : initVertices_) { Row row; List edgeList; edgeList.values.emplace_back(vertex); @@ -297,8 +283,8 @@ folly::Future TraverseExecutor::buildResult() { return finish(ResultBuilder().value(Value(std::move(result_))).build()); } if (FLAGS_max_job_size <= 1) { - for (const auto& vid : initVids_) { - auto paths = buildPath(vid, minStep, maxStep); + for (const auto& initVertex : initVertices_) { + auto paths = buildPath(initVertex, minStep, maxStep); if (paths.empty()) { continue; } @@ -312,22 +298,22 @@ folly::Future TraverseExecutor::buildResult() { } folly::Future TraverseExecutor::buildPathMultiJobs(size_t minStep, size_t maxStep) { - DataSet vids; - vids.rows.reserve(initVids_.size()); - for (auto& vid : initVids_) { + DataSet vertices; + vertices.rows.reserve(initVertices_.size()); + for (auto& initVertex : initVertices_) { Row row; - row.values.emplace_back(std::move(vid)); - vids.rows.emplace_back(std::move(row)); + row.values.emplace_back(std::move(initVertex)); + vertices.rows.emplace_back(std::move(row)); } - auto val = std::make_shared(std::move(vids)); + auto val = std::make_shared(std::move(vertices)); auto iter = std::make_unique(val); auto scatter = [this, minStep, maxStep]( size_t begin, size_t end, Iterator* tmpIter) mutable -> std::vector { std::vector rows; for (; tmpIter->valid() && begin++ < end; tmpIter->next()) { - auto& vid = tmpIter->getColumn(0); - auto paths = buildPath(vid, minStep, maxStep); + auto& initVertex = tmpIter->getColumn(0); + auto paths = buildPath(initVertex, minStep, maxStep); if (paths.empty()) { continue; } @@ -355,8 +341,10 @@ folly::Future TraverseExecutor::buildPathMultiJobs(size_t minStep, size_ } // build path based on BFS through adjancency list -std::vector TraverseExecutor::buildPath(const Value& vid, size_t minStep, size_t maxStep) { - auto vidIter = adjList_.find(vid); +std::vector TraverseExecutor::buildPath(const Value& initVertex, + size_t minStep, + size_t maxStep) { + auto vidIter = adjList_.find(initVertex); if (vidIter == adjList_.end()) { return std::vector(); } @@ -380,7 +368,7 @@ std::vector TraverseExecutor::buildPath(const Value& vid, size_t minStep, s if (maxStep == 1) { if (traverse_->trackPrevPath()) { std::vector newResult; - auto dstIter = dst2PathsMap_.find(vid); + auto dstIter = dst2PathsMap_.find(initVertex); if (dstIter == dst2PathsMap_.end()) { return std::vector(); } @@ -461,7 +449,7 @@ std::vector TraverseExecutor::buildPath(const Value& vid, size_t minStep, s } if (traverse_->trackPrevPath()) { std::vector newPaths; - auto dstIter = dst2PathsMap_.find(vid); + auto dstIter = dst2PathsMap_.find(initVertex); if (dstIter != dst2PathsMap_.end()) { auto& prevPaths = dstIter->second; for (auto& prevPath : prevPaths) { diff --git a/src/graph/executor/query/TraverseExecutor.h b/src/graph/executor/query/TraverseExecutor.h index dbdd14d3caf..ed4ea9bf800 100644 --- a/src/graph/executor/query/TraverseExecutor.h +++ b/src/graph/executor/query/TraverseExecutor.h @@ -62,7 +62,7 @@ class TraverseExecutor final : public StorageAccessExecutor { folly::Future buildResult(); - std::vector buildPath(const Value& vid, size_t minStep, size_t maxStep); + std::vector buildPath(const Value& initVertex, size_t minStep, size_t maxStep); folly::Future buildPathMultiJobs(size_t minStep, size_t maxStep); @@ -77,7 +77,7 @@ class TraverseExecutor final : public StorageAccessExecutor { bool hasSameEdge(const std::vector& edgeList, const Edge& edge); - std::vector buildZeroStepPath(GetNeighborsIter* iter); + std::vector buildZeroStepPath(); Expression* selectFilter(); @@ -127,7 +127,7 @@ class TraverseExecutor final : public StorageAccessExecutor { ObjectPool objPool_; std::vector vids_; - std::vector initVids_; + std::vector initVertices_; DataSet result_; // Key : vertex Value : adjacent edges std::unordered_map, VertexHash, VertexEqual> adjList_; From ec79402801feced9851f068505b85e8528f70f4c Mon Sep 17 00:00:00 2001 From: "jie.wang" <38901892+jievince@users.noreply.github.com> Date: Tue, 3 Jan 2023 16:51:06 +0800 Subject: [PATCH 02/51] revert strange return (#5183) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/parser/parser.yy | 13 +++++++++---- tests/tck/features/set/Set.feature | 22 +++++++++++++++++----- tests/tck/features/yield/return.feature | 20 ++++++++++++++++++-- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 68a7ed3d478..966e81c6f77 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -1644,6 +1644,14 @@ yield_sentence s->setWhereClause($4); $$ = s; } + | KW_RETURN yield_columns { + auto *s = new YieldSentence($2); + $$ = s; + } + | KW_RETURN KW_DISTINCT yield_columns { + auto *s = new YieldSentence($3, true); + $$ = s; + } ; unwind_clause @@ -1720,10 +1728,7 @@ reading_with_clauses ; match_sentence - : match_return { - $$ = new MatchSentence(new MatchClauseList(), $1); - } - | reading_clauses match_return { + : reading_clauses match_return { $$ = new MatchSentence($1, $2); } | reading_with_clauses match_return { diff --git a/tests/tck/features/set/Set.feature b/tests/tck/features/set/Set.feature index 44f8a93b52e..7e8ce37f1a0 100644 --- a/tests/tck/features/set/Set.feature +++ b/tests/tck/features/set/Set.feature @@ -75,12 +75,21 @@ Feature: Set Test | a | | 1 | | 2 | - # cypher doesn't support intersect + # The standalone return statement is not a cypher statement but a ngql statement in nebula... + # So it can't be mixed with other cypher statements When executing query: """ UNWIND [1,2] AS a RETURN a INTERSECT - RETURN 2 AS a + RETURN a + """ + Then a SyntaxError should be raised at runtime: + When executing query: + """ + UNWIND [1,2] AS a RETURN a + INTERSECT + WITH 2 AS a + RETURN a """ Then the result should be, in any order: | a | @@ -89,7 +98,8 @@ Feature: Set Test """ UNWIND [1,2,3] AS a RETURN a, 100 INTERSECT - RETURN 2 AS a, 100 + WITH 2 AS a + RETURN a, 100 """ Then the result should be, in any order: | a | 100 | @@ -291,7 +301,8 @@ Feature: Set Test """ UNWIND [1,2,3] AS a RETURN a MINUS - RETURN 4 AS a + WITH 4 AS a + RETURN a """ Then the result should be, in any order, with relax comparison: | a | @@ -302,7 +313,8 @@ Feature: Set Test """ UNWIND [1,2,3] AS a RETURN a MINUS - RETURN 2 AS a + WITH 2 AS a + RETURN a """ Then the result should be, in any order, with relax comparison: | a | diff --git a/tests/tck/features/yield/return.feature b/tests/tck/features/yield/return.feature index 11070dd6974..468cbe93000 100644 --- a/tests/tck/features/yield/return.feature +++ b/tests/tck/features/yield/return.feature @@ -1,7 +1,7 @@ # Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License. -Feature: Return +Feature: Return. A standalone return sentence is actually a yield sentence Background: Given a graph with space named "nba" @@ -21,6 +21,22 @@ Feature: Return Then the result should be, in any order: | (1+1) | "1+1" | (INT)3.14 | (STRING)(1+1) | (STRING)true | | 2 | "1+1" | 3 | "2" | "true" | + When executing query: + """ + GO FROM "Tony Parker" OVER like YIELD id($$) AS vid | RETURN $-.vid AS dst + """ + Then the result should be, in any order, with relax comparison: + | dst | + | "LaMarcus Aldridge" | + | "Manu Ginobili" | + | "Tim Duncan" | + When executing query: + """ + FETCH PROP ON player "Tony Parker" YIELD player.age as age | RETURN $-.age + 100 AS age + """ + Then the result should be, in any order, with relax comparison: + | age | + | 136 | Scenario: hash call When executing query: @@ -79,4 +95,4 @@ Feature: Return """ RETURN name """ - Then a SemanticError should be raised at runtime: Alias used but not defined: `name' + Then a SemanticError should be raised at runtime: Invalid label identifiers: name From 008bc8a7d4c4499129e28f7045c2c7f061b384e8 Mon Sep 17 00:00:00 2001 From: "Harris.Chu" <1726587+HarrisChu@users.noreply.github.com> Date: Wed, 4 Jan 2023 11:52:34 +0800 Subject: [PATCH 03/51] fix stderr save error log (#5188) --- conf/nebula-graphd.conf.default | 2 +- conf/nebula-graphd.conf.production | 2 +- conf/nebula-metad.conf.default | 2 +- conf/nebula-metad.conf.production | 2 +- conf/nebula-standalone.conf.default | 2 +- conf/nebula-storaged-listener.conf.default | 2 +- conf/nebula-storaged-listener.conf.production | 2 +- conf/nebula-storaged.conf.default | 2 +- conf/nebula-storaged.conf.production | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default index f6eeb7549df..feb9e273a21 100644 --- a/conf/nebula-graphd.conf.default +++ b/conf/nebula-graphd.conf.default @@ -27,7 +27,7 @@ --stdout_log_file=graphd-stdout.log --stderr_log_file=graphd-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # wether logging files' name contain time stamp. --timestamp_in_logfile_name=true ########## query ########## diff --git a/conf/nebula-graphd.conf.production b/conf/nebula-graphd.conf.production index e7cfbf3d810..d5b07642394 100644 --- a/conf/nebula-graphd.conf.production +++ b/conf/nebula-graphd.conf.production @@ -25,7 +25,7 @@ --stdout_log_file=graphd-stdout.log --stderr_log_file=graphd-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # wether logging files' name contain timestamp --timestamp_in_logfile_name=true diff --git a/conf/nebula-metad.conf.default b/conf/nebula-metad.conf.default index ce9903200a8..89b594f36f5 100644 --- a/conf/nebula-metad.conf.default +++ b/conf/nebula-metad.conf.default @@ -19,7 +19,7 @@ --stdout_log_file=metad-stdout.log --stderr_log_file=metad-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # wether logging files' name contain time stamp, If Using logrotate to rotate logging files, than should set it to true. --timestamp_in_logfile_name=true diff --git a/conf/nebula-metad.conf.production b/conf/nebula-metad.conf.production index 84b18f831d4..3ea93ffac0c 100644 --- a/conf/nebula-metad.conf.production +++ b/conf/nebula-metad.conf.production @@ -19,7 +19,7 @@ --stdout_log_file=metad-stdout.log --stderr_log_file=metad-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # wether logging files' name contain time stamp. --timestamp_in_logfile_name=true diff --git a/conf/nebula-standalone.conf.default b/conf/nebula-standalone.conf.default index 16694304c60..e5f7a449efe 100644 --- a/conf/nebula-standalone.conf.default +++ b/conf/nebula-standalone.conf.default @@ -27,7 +27,7 @@ --stdout_log_file=standalone-stdout.log --stderr_log_file=standalone-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 ########## query ########## # Whether to treat partial success as an error. diff --git a/conf/nebula-storaged-listener.conf.default b/conf/nebula-storaged-listener.conf.default index 02936b7d192..86d74d3c356 100644 --- a/conf/nebula-storaged-listener.conf.default +++ b/conf/nebula-storaged-listener.conf.default @@ -22,7 +22,7 @@ --stdout_log_file=storaged-listener-stdout.log --stderr_log_file=storaged-listener-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # Wether logging files' name contain timestamp. --timestamp_in_logfile_name=true diff --git a/conf/nebula-storaged-listener.conf.production b/conf/nebula-storaged-listener.conf.production index a8c33d5ba07..72467e7fab1 100644 --- a/conf/nebula-storaged-listener.conf.production +++ b/conf/nebula-storaged-listener.conf.production @@ -22,7 +22,7 @@ --stdout_log_file=storaged-listener-stdout.log --stderr_log_file=storaged-listener-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # Wether logging files' name contain timestamp. --timestamp_in_logfile_name=true diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default index c41bbf1cb8e..34c345ac9ed 100644 --- a/conf/nebula-storaged.conf.default +++ b/conf/nebula-storaged.conf.default @@ -21,7 +21,7 @@ --stdout_log_file=storaged-stdout.log --stderr_log_file=storaged-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # Wether logging files' name contain time stamp. --timestamp_in_logfile_name=true diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production index 0c1bcd34a62..d0004f06fc5 100644 --- a/conf/nebula-storaged.conf.production +++ b/conf/nebula-storaged.conf.production @@ -21,7 +21,7 @@ --stdout_log_file=storaged-listener-stdout.log --stderr_log_file=storaged-listener-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 +--stderrthreshold=3 # Wether logging files' name contain timestamp. --timestamp_in_logfile_name=true From a5a578e3e90eb3d22ad3991cc0302cdb836423cb Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Wed, 4 Jan 2023 12:14:48 +0800 Subject: [PATCH 04/51] fix processor_test timeout (#5180) * fix processor_test timeout * ... Co-authored-by: Yee <2520865+yixinglu@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/meta/MetaServiceUtils.cpp | 5 +- .../processors/schema/AlterEdgeProcessor.cpp | 2 +- .../processors/schema/AlterTagProcessor.cpp | 2 +- src/meta/test/ProcessorTest.cpp | 57 +++++++++++++++---- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/meta/MetaServiceUtils.cpp b/src/meta/MetaServiceUtils.cpp index ef2fe06fe2c..4a23439dc1e 100644 --- a/src/meta/MetaServiceUtils.cpp +++ b/src/meta/MetaServiceUtils.cpp @@ -48,7 +48,10 @@ bool isLegalTypeConversion(cpp2::ColumnTypeDef from, cpp2::ColumnTypeDef to) { if (to.get_type() == nebula::cpp2::PropertyType::STRING) { return true; } else if (to.get_type() == nebula::cpp2::PropertyType::FIXED_STRING) { - return from.get_type_length() <= to.get_type_length(); + if (!from.type_length_ref().has_value() || !to.type_length_ref().has_value()) { + return false; + } + return *from.type_length_ref() <= *to.type_length_ref(); } else { return false; } diff --git a/src/meta/processors/schema/AlterEdgeProcessor.cpp b/src/meta/processors/schema/AlterEdgeProcessor.cpp index ccf7b899b4e..84ad4310064 100644 --- a/src/meta/processors/schema/AlterEdgeProcessor.cpp +++ b/src/meta/processors/schema/AlterEdgeProcessor.cpp @@ -125,7 +125,7 @@ void AlterEdgeProcessor::process(const cpp2::AlterEdgeReq& req) { auto& cols = edgeItem.get_schema().get_columns(); for (auto& col : cols) { auto retCode = MetaServiceUtils::alterColumnDefs( - columns, prop, col, *edgeItem.op_ref(), std::move(allVersionedColumns), true); + columns, prop, col, *edgeItem.op_ref(), allVersionedColumns, true); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(INFO) << "Alter edge column error " << apache::thrift::util::enumNameSafe(retCode); handleErrorCode(retCode); diff --git a/src/meta/processors/schema/AlterTagProcessor.cpp b/src/meta/processors/schema/AlterTagProcessor.cpp index 1a2300ccebd..75997abef85 100644 --- a/src/meta/processors/schema/AlterTagProcessor.cpp +++ b/src/meta/processors/schema/AlterTagProcessor.cpp @@ -122,7 +122,7 @@ void AlterTagProcessor::process(const cpp2::AlterTagReq& req) { auto& cols = tagItem.get_schema().get_columns(); for (auto& col : cols) { auto retCode = MetaServiceUtils::alterColumnDefs( - columns, prop, col, *tagItem.op_ref(), std::move(allVersionedColumns)); + columns, prop, col, *tagItem.op_ref(), allVersionedColumns); if (retCode != nebula::cpp2::ErrorCode::SUCCEEDED) { LOG(INFO) << "Alter tag column error " << apache::thrift::util::enumNameSafe(retCode); handleErrorCode(retCode); diff --git a/src/meta/test/ProcessorTest.cpp b/src/meta/test/ProcessorTest.cpp index 36107d2ee8c..367cebfda1c 100644 --- a/src/meta/test/ProcessorTest.cpp +++ b/src/meta/test/ProcessorTest.cpp @@ -1783,11 +1783,28 @@ TEST(ProcessorTest, AlterTagTest) { req.space_id_ref() = 1; req.tag_name_ref() = "tag_0"; req.tag_items_ref() = items; - auto* processor = AlterTagProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_UNSUPPORTED, resp.get_code()); + + { + auto* processor = AlterTagProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_UNSUPPORTED, resp.get_code()); + } + { + req.tag_items_ref() + ->back() + .schema_ref() + ->columns_ref() + ->back() + .type_ref() + ->type_length_ref() = 6; + auto* processor = AlterTagProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } } } @@ -2376,11 +2393,27 @@ TEST(ProcessorTest, AlterEdgeTest) { req.space_id_ref() = 1; req.edge_name_ref() = "edge_0"; req.edge_items_ref() = items; - auto* processor = AlterEdgeProcessor::instance(kv.get()); - auto f = processor->getFuture(); - processor->process(req); - auto resp = std::move(f).get(); - ASSERT_EQ(nebula::cpp2::ErrorCode::E_UNSUPPORTED, resp.get_code()); + { + auto* processor = AlterEdgeProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::E_UNSUPPORTED, resp.get_code()); + } + { + req.edge_items_ref() + ->back() + .schema_ref() + ->columns_ref() + ->back() + .type_ref() + ->type_length_ref() = 6; + auto* processor = AlterEdgeProcessor::instance(kv.get()); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + ASSERT_EQ(nebula::cpp2::ErrorCode::SUCCEEDED, resp.get_code()); + } } } @@ -2389,7 +2422,7 @@ TEST(ProcessorTest, AlterTagForMoreThan256TimesTest) { auto kv = MockCluster::initMetaKV(rootPath.path()); TestUtils::assembleSpace(kv.get(), 1, 1); TestUtils::mockTag(kv.get(), 1); - const int times = 1000; + const int times = 300; int totalScheVer = 1; std::vector expectedCols; for (auto i = 0; i < 2; i++) { @@ -2516,7 +2549,7 @@ TEST(ProcessorTest, AlterEdgeForMoreThan256TimesTest) { auto kv = MockCluster::initMetaKV(rootPath.path()); TestUtils::assembleSpace(kv.get(), 1, 1); TestUtils::mockEdge(kv.get(), 1); - const int times = 1000; + const int times = 300; int totalScheVer = 1; std::vector expectedCols; for (auto i = 0; i < 2; i++) { From b465141627b8f382e97a9f03093d4ca3101ad492 Mon Sep 17 00:00:00 2001 From: canon <87342612+caton-hpg@users.noreply.github.com> Date: Wed, 4 Jan 2023 13:44:40 +0800 Subject: [PATCH 05/51] fix error code (#5186) --- src/interface/common.thrift | 58 ++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/src/interface/common.thrift b/src/interface/common.thrift index 961199741c8..0bdf5d44f94 100644 --- a/src/interface/common.thrift +++ b/src/interface/common.thrift @@ -325,6 +325,9 @@ enum ErrorCode { E_USER_NOT_FOUND = -18, // User does not exist E_STATS_NOT_FOUND = -19, // Statistics do not exist E_SERVICE_NOT_FOUND = -20, // No current service found + E_DRAINER_NOT_FOUND = -21, // Drainer does not exist[only ent] + E_DRAINER_CLIENT_NOT_FOUND = -22, // Drainer client does not exist[only ent] + E_PART_STOPPED = -23, // The current partition has already been stopped[only ent] // backup failed E_BACKUP_FAILED = -24, // Backup failed @@ -334,6 +337,13 @@ enum ErrorCode { E_REBUILD_INDEX_FAILED = -28, // Index rebuild failed E_INVALID_PASSWORD = -29, // Password is invalid E_FAILED_GET_ABS_PATH = -30, // Unable to get absolute path + E_LISTENER_PROGRESS_FAILED = -31, // Get listener sync progress failed[only ent] + E_SYNC_LISTENER_NOT_FOUND = -32, // Sync Listener does not exist[only ent] + E_DRAINER_PROGRESS_FAILED = -33, // Get drainer sync progress failed[only ent] + + E_PART_DISABLED = -34, // [only ent] + E_PART_ALREADY_STARTED = -35, // [only ent] + E_PART_ALREADY_STOPPED = -36, // [only ent] // 1xxx for graphd E_BAD_USERNAME_PASSWORD = -1001, // Authentication failed @@ -359,12 +369,13 @@ enum ErrorCode { E_CONFLICT = -2008, // Parameters conflict with meta data E_INVALID_PARM = -2009, // Invalid parameter E_WRONGCLUSTER = -2010, // Wrong cluster - E_ZONE_NOT_ENOUGH = -2011, // Listener conflicts - E_ZONE_IS_EMPTY = -2012, // Host not exist - E_SCHEMA_NAME_EXISTS = -2013, // Schema name already exists - E_RELATED_INDEX_EXISTS = -2014, // There are still indexes related to tag or edge, cannot drop it - E_RELATED_SPACE_EXISTS = -2015, // There are still some space on the host, cannot drop it - E_RELATED_FULLTEXT_INDEX_EXISTS = -2016, // There are still fulltext index on tag/edge + E_ZONE_NOT_ENOUGH = -2011, // Host is not enough + E_ZONE_IS_EMPTY = -2012, // Host does not exist + E_LISTENER_CONFLICT = -2013, // Listener conflicts[only ent] + E_SCHEMA_NAME_EXISTS = -2014, // Schema name already exists + E_RELATED_INDEX_EXISTS = -2015, // There are still indexes related to tag or edge, cannot drop it + E_RELATED_SPACE_EXISTS = -2016, // There are still some space on the host, cannot drop it + E_RELATED_FULLTEXT_INDEX_EXISTS = -2017, // There are still fulltext index on tag/edge E_STORE_FAILURE = -2021, // Failed to store data E_STORE_SEGMENT_ILLEGAL = -2022, // Illegal storage segment @@ -374,7 +385,7 @@ enum ErrorCode { E_NO_VALID_HOST = -2026, // Lack of valid hosts E_CORRUPTED_BALANCE_PLAN = -2027, // A data balancing plan that has been corrupted E_NO_INVALID_BALANCE_PLAN = -2028, // No invalid balance plan - E_HOST_CAN_NOT_BE_ADDED = -2029, // the host can not be added for it's not a storage host + E_NO_VALID_DRAINER = -2029, // Lack of valid drainers[only ent] // Authentication Failure @@ -384,6 +395,10 @@ enum ErrorCode { E_INVALID_CHARSET = -2033, // Invalid character set E_INVALID_COLLATE = -2034, // Invalid character sorting rules E_CHARSET_COLLATE_NOT_MATCH = -2035, // Character set and character sorting rule mismatch + E_PRIVILEGE_ALL_TAG_EDGE_SETTLED = -2036, // drop all tag/edge before do some grant/revoke[only ent] + E_PRIVILEGE_NOT_EXIST = -2037, // remove un-exist privilege[only ent] + E_PRIVILEGE_NEED_BASIC_ROLE = -2038, // only basic role support tag/edge privilege[only ent] + E_PRIVILEGE_ACTION_INVALID = -2039, // only add and drop now.[only ent] // Admin Failure E_SNAPSHOT_FAILURE = -2040, // Failed to generate a snapshot @@ -423,7 +438,11 @@ enum ErrorCode { E_QUERY_NOT_FOUND = -2073, // Query not found E_AGENT_HB_FAILUE = -2074, // Failed to receive heartbeat from agent - E_ACCESS_ES_FAILURE = -2080, // Failed to access elasticsearch + E_INVALID_VARIABLE = -2080, // [only ent] + E_VARIABLE_TYPE_VALUE_MISMATCH = -2081, // [only ent] + E_HOST_CAN_NOT_BE_ADDED = -2082, // the host can not be added for it's not a storage host + + E_ACCESS_ES_FAILURE = -2090, // Failed to access elasticsearch E_GRAPH_MEMORY_EXCEEDED = -2600, // Graph memory exceeded @@ -516,6 +535,29 @@ enum ErrorCode { E_RAFT_BUFFER_OVERFLOW = -3529, // Cache overflow E_RAFT_ATOMIC_OP_FAILED = -3530, // Atomic operation failed E_LEADER_LEASE_FAILED = -3531, // Leader lease expired + E_RAFT_CAUGHT_UP = -3532, // Data has been synchronized on Raft[only ent] + + // 4xxx for drainer + E_LOG_GAP = -4001, // Drainer logs lag behind[only ent] + E_LOG_STALE = -4002, // Drainer logs are out of date[only ent] + E_INVALID_DRAINER_STORE = -4003, // The drainer data storage is invalid[only ent] + E_SPACE_MISMATCH = -4004, // Graph space mismatch[only ent] + E_PART_MISMATCH = -4005, // Partition mismatch[only ent] + E_DATA_CONFLICT = -4006, // Data conflict[only ent] + E_REQ_CONFLICT = -4007, // Request conflict[only ent] + E_DATA_ILLEGAL = -4008, // Illegal data[only ent] + + // 5xxx for cache + E_CACHE_CONFIG_ERROR = -5001, // Cache configuration error[only ent] + E_NOT_ENOUGH_SPACE = -5002, // Insufficient space[only ent] + E_CACHE_MISS = -5003, // No cache hit[only ent] + E_POOL_NOT_FOUND = -5004, // [only ent] + E_CACHE_WRITE_FAILURE = -5005, // Write cache failed[only ent] + + // 7xxx for nebula enterprise + // license related + E_NODE_NUMBER_EXCEED_LIMIT = -7001, // Number of machines exceeded the limit[only ent] + E_PARSING_LICENSE_FAILURE = -7002, // Failed to resolve certificate[only ent] E_STORAGE_MEMORY_EXCEEDED = -3600, // Storage memory exceeded From 8ddfc03905b4b7f9993f119666761c91e8a69708 Mon Sep 17 00:00:00 2001 From: Cheng Xuntao <7731943+xtcyclist@users.noreply.github.com> Date: Wed, 4 Jan 2023 19:22:50 +0800 Subject: [PATCH 06/51] rename the "test" space to "ngdata". (#5197) * rename the "test" space to "ngdata". * add ngdata --- tests/conftest.py | 4 ++-- tests/data/{test => ngdata}/Edge_label_limits.csv | 0 tests/data/{test => ngdata}/Edge_ttl_0.csv | 0 tests/data/{test => ngdata}/Label_0.csv | 0 tests/data/{test => ngdata}/Label_1.csv | 0 tests/data/{test => ngdata}/Label_10.csv | 0 tests/data/{test => ngdata}/Label_11.csv | 0 tests/data/{test => ngdata}/Label_2.csv | 0 tests/data/{test => ngdata}/Label_3.csv | 0 tests/data/{test => ngdata}/Label_4.csv | 0 tests/data/{test => ngdata}/Label_5.csv | 0 tests/data/{test => ngdata}/Label_6.csv | 0 tests/data/{test => ngdata}/Label_7.csv | 0 tests/data/{test => ngdata}/Label_8.csv | 0 tests/data/{test => ngdata}/Label_9.csv | 0 tests/data/{test => ngdata}/Label_same_prop_0.csv | 0 tests/data/{test => ngdata}/Label_same_prop_1.csv | 0 tests/data/{test => ngdata}/Label_ttl_0.csv | 0 tests/data/{test => ngdata}/README.md | 0 tests/data/{test => ngdata}/Rel_0.csv | 0 tests/data/{test => ngdata}/Rel_1.csv | 0 tests/data/{test => ngdata}/Rel_2.csv | 0 tests/data/{test => ngdata}/Rel_3.csv | 0 tests/data/{test => ngdata}/Rel_4.csv | 0 tests/data/{test => ngdata}/Rel_5.csv | 0 tests/data/{test => ngdata}/config.yaml | 2 +- tests/data/{test => ngdata}/gen_graph.sh | 0 tests/tck/conftest.py | 6 +++--- tests/tck/features/expression/Attribute1.feature | 2 +- tests/tck/features/match/Path.feature | 2 +- tests/tck/features/optimizer/CasesUsingTestSpace.feature | 2 +- 31 files changed, 9 insertions(+), 9 deletions(-) rename tests/data/{test => ngdata}/Edge_label_limits.csv (100%) rename tests/data/{test => ngdata}/Edge_ttl_0.csv (100%) rename tests/data/{test => ngdata}/Label_0.csv (100%) rename tests/data/{test => ngdata}/Label_1.csv (100%) rename tests/data/{test => ngdata}/Label_10.csv (100%) rename tests/data/{test => ngdata}/Label_11.csv (100%) rename tests/data/{test => ngdata}/Label_2.csv (100%) rename tests/data/{test => ngdata}/Label_3.csv (100%) rename tests/data/{test => ngdata}/Label_4.csv (100%) rename tests/data/{test => ngdata}/Label_5.csv (100%) rename tests/data/{test => ngdata}/Label_6.csv (100%) rename tests/data/{test => ngdata}/Label_7.csv (100%) rename tests/data/{test => ngdata}/Label_8.csv (100%) rename tests/data/{test => ngdata}/Label_9.csv (100%) rename tests/data/{test => ngdata}/Label_same_prop_0.csv (100%) rename tests/data/{test => ngdata}/Label_same_prop_1.csv (100%) rename tests/data/{test => ngdata}/Label_ttl_0.csv (100%) rename tests/data/{test => ngdata}/README.md (100%) rename tests/data/{test => ngdata}/Rel_0.csv (100%) rename tests/data/{test => ngdata}/Rel_1.csv (100%) rename tests/data/{test => ngdata}/Rel_2.csv (100%) rename tests/data/{test => ngdata}/Rel_3.csv (100%) rename tests/data/{test => ngdata}/Rel_4.csv (100%) rename tests/data/{test => ngdata}/Rel_5.csv (100%) rename tests/data/{test => ngdata}/config.yaml (99%) rename tests/data/{test => ngdata}/gen_graph.sh (100%) diff --git a/tests/conftest.py b/tests/conftest.py index 39ab47788d5..7ca04276f49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -232,8 +232,8 @@ def load_student_data(): yield load_csv_data_once("student") @pytest.fixture(scope="session") -def load_test_data(): - yield load_csv_data_once("test") +def load_ngdata_data(): + yield load_csv_data_once("ngdata") # TODO(yee): Delete this when we migrate all test cases @pytest.fixture(scope="class") diff --git a/tests/data/test/Edge_label_limits.csv b/tests/data/ngdata/Edge_label_limits.csv similarity index 100% rename from tests/data/test/Edge_label_limits.csv rename to tests/data/ngdata/Edge_label_limits.csv diff --git a/tests/data/test/Edge_ttl_0.csv b/tests/data/ngdata/Edge_ttl_0.csv similarity index 100% rename from tests/data/test/Edge_ttl_0.csv rename to tests/data/ngdata/Edge_ttl_0.csv diff --git a/tests/data/test/Label_0.csv b/tests/data/ngdata/Label_0.csv similarity index 100% rename from tests/data/test/Label_0.csv rename to tests/data/ngdata/Label_0.csv diff --git a/tests/data/test/Label_1.csv b/tests/data/ngdata/Label_1.csv similarity index 100% rename from tests/data/test/Label_1.csv rename to tests/data/ngdata/Label_1.csv diff --git a/tests/data/test/Label_10.csv b/tests/data/ngdata/Label_10.csv similarity index 100% rename from tests/data/test/Label_10.csv rename to tests/data/ngdata/Label_10.csv diff --git a/tests/data/test/Label_11.csv b/tests/data/ngdata/Label_11.csv similarity index 100% rename from tests/data/test/Label_11.csv rename to tests/data/ngdata/Label_11.csv diff --git a/tests/data/test/Label_2.csv b/tests/data/ngdata/Label_2.csv similarity index 100% rename from tests/data/test/Label_2.csv rename to tests/data/ngdata/Label_2.csv diff --git a/tests/data/test/Label_3.csv b/tests/data/ngdata/Label_3.csv similarity index 100% rename from tests/data/test/Label_3.csv rename to tests/data/ngdata/Label_3.csv diff --git a/tests/data/test/Label_4.csv b/tests/data/ngdata/Label_4.csv similarity index 100% rename from tests/data/test/Label_4.csv rename to tests/data/ngdata/Label_4.csv diff --git a/tests/data/test/Label_5.csv b/tests/data/ngdata/Label_5.csv similarity index 100% rename from tests/data/test/Label_5.csv rename to tests/data/ngdata/Label_5.csv diff --git a/tests/data/test/Label_6.csv b/tests/data/ngdata/Label_6.csv similarity index 100% rename from tests/data/test/Label_6.csv rename to tests/data/ngdata/Label_6.csv diff --git a/tests/data/test/Label_7.csv b/tests/data/ngdata/Label_7.csv similarity index 100% rename from tests/data/test/Label_7.csv rename to tests/data/ngdata/Label_7.csv diff --git a/tests/data/test/Label_8.csv b/tests/data/ngdata/Label_8.csv similarity index 100% rename from tests/data/test/Label_8.csv rename to tests/data/ngdata/Label_8.csv diff --git a/tests/data/test/Label_9.csv b/tests/data/ngdata/Label_9.csv similarity index 100% rename from tests/data/test/Label_9.csv rename to tests/data/ngdata/Label_9.csv diff --git a/tests/data/test/Label_same_prop_0.csv b/tests/data/ngdata/Label_same_prop_0.csv similarity index 100% rename from tests/data/test/Label_same_prop_0.csv rename to tests/data/ngdata/Label_same_prop_0.csv diff --git a/tests/data/test/Label_same_prop_1.csv b/tests/data/ngdata/Label_same_prop_1.csv similarity index 100% rename from tests/data/test/Label_same_prop_1.csv rename to tests/data/ngdata/Label_same_prop_1.csv diff --git a/tests/data/test/Label_ttl_0.csv b/tests/data/ngdata/Label_ttl_0.csv similarity index 100% rename from tests/data/test/Label_ttl_0.csv rename to tests/data/ngdata/Label_ttl_0.csv diff --git a/tests/data/test/README.md b/tests/data/ngdata/README.md similarity index 100% rename from tests/data/test/README.md rename to tests/data/ngdata/README.md diff --git a/tests/data/test/Rel_0.csv b/tests/data/ngdata/Rel_0.csv similarity index 100% rename from tests/data/test/Rel_0.csv rename to tests/data/ngdata/Rel_0.csv diff --git a/tests/data/test/Rel_1.csv b/tests/data/ngdata/Rel_1.csv similarity index 100% rename from tests/data/test/Rel_1.csv rename to tests/data/ngdata/Rel_1.csv diff --git a/tests/data/test/Rel_2.csv b/tests/data/ngdata/Rel_2.csv similarity index 100% rename from tests/data/test/Rel_2.csv rename to tests/data/ngdata/Rel_2.csv diff --git a/tests/data/test/Rel_3.csv b/tests/data/ngdata/Rel_3.csv similarity index 100% rename from tests/data/test/Rel_3.csv rename to tests/data/ngdata/Rel_3.csv diff --git a/tests/data/test/Rel_4.csv b/tests/data/ngdata/Rel_4.csv similarity index 100% rename from tests/data/test/Rel_4.csv rename to tests/data/ngdata/Rel_4.csv diff --git a/tests/data/test/Rel_5.csv b/tests/data/ngdata/Rel_5.csv similarity index 100% rename from tests/data/test/Rel_5.csv rename to tests/data/ngdata/Rel_5.csv diff --git a/tests/data/test/config.yaml b/tests/data/ngdata/config.yaml similarity index 99% rename from tests/data/test/config.yaml rename to tests/data/ngdata/config.yaml index a9c0ad2bc56..84c6b6cd37e 100644 --- a/tests/data/test/config.yaml +++ b/tests/data/ngdata/config.yaml @@ -1,5 +1,5 @@ space: - name: test + name: ngdata partitionNum: 1 ReplicaFactor: 1 vidType: int diff --git a/tests/data/test/gen_graph.sh b/tests/data/ngdata/gen_graph.sh similarity index 100% rename from tests/data/test/gen_graph.sh rename to tests/data/ngdata/gen_graph.sh diff --git a/tests/tck/conftest.py b/tests/tck/conftest.py index 2d604689fe4..3895e98285a 100644 --- a/tests/tck/conftest.py +++ b/tests/tck/conftest.py @@ -182,7 +182,7 @@ def preload_space( load_nba_int_vid_data, load_student_data, load_ldbc_v0_3_3, - load_test_data, + load_ngdata_data, exec_ctx, ): space = normalize_outline_scenario(request, space) @@ -194,8 +194,8 @@ def preload_space( exec_ctx["space_desc"] = load_student_data elif space == "ldbc_v0_3_3": exec_ctx["ldbc_v0_3_3"] = load_ldbc_v0_3_3 - elif space == "test": - exec_ctx["space_desc"] = load_test_data + elif space == "ngdata": + exec_ctx["space_desc"] = load_ngdata_data else: raise ValueError(f"Invalid space name given: {space}") diff --git a/tests/tck/features/expression/Attribute1.feature b/tests/tck/features/expression/Attribute1.feature index 91b2e459f62..92cfcf9387a 100644 --- a/tests/tck/features/expression/Attribute1.feature +++ b/tests/tck/features/expression/Attribute1.feature @@ -4,7 +4,7 @@ Feature: Attribute using test Background: - Given load "test" csv data to a new space + Given a graph with space named "ngdata" Scenario: Attribute with null data When executing query: diff --git a/tests/tck/features/match/Path.feature b/tests/tck/features/match/Path.feature index ea2ec112b8a..80f9fb698ee 100644 --- a/tests/tck/features/match/Path.feature +++ b/tests/tck/features/match/Path.feature @@ -4,7 +4,7 @@ Feature: Matching paths Background: - Given load "test" csv data to a new space + Given a graph with space named "ngdata" Scenario: distinct edges and paths When executing query: diff --git a/tests/tck/features/optimizer/CasesUsingTestSpace.feature b/tests/tck/features/optimizer/CasesUsingTestSpace.feature index d1029309f9b..2e27df659fd 100644 --- a/tests/tck/features/optimizer/CasesUsingTestSpace.feature +++ b/tests/tck/features/optimizer/CasesUsingTestSpace.feature @@ -4,7 +4,7 @@ Feature: Push Filter Down Cases Using the test Space Background: - Given load "test" csv data to a new space + Given a graph with space named "ngdata" Scenario: push filter down hash join bug fix When profiling query: From 175028303b7b65db69e28926057121b4fcd7e566 Mon Sep 17 00:00:00 2001 From: Cheng Xuntao <7731943+xtcyclist@users.noreply.github.com> Date: Thu, 5 Jan 2023 11:04:21 +0800 Subject: [PATCH 07/51] Revise the usages of FATAL, DFATAL, LOG, DLOG. (#5181) * Revise the usages of FATAL, DFATAL, LOG, DLOG. * fix. * revise dfatal. --- src/clients/meta/MetaClient.cpp | 2 +- src/codec/RowReaderV1.cpp | 3 +- src/codec/RowReaderV2.cpp | 3 +- src/codec/RowReaderWrapper.cpp | 2 +- src/codec/RowWriterV2.cpp | 10 +++---- src/common/base/Arena.cpp | 2 +- src/common/base/Status.cpp | 2 +- src/common/datatypes/Value.cpp | 26 ++++++++-------- .../expression/ArithmeticExpression.cpp | 6 ++-- src/common/expression/Expression.cpp | 10 +++---- .../expression/LabelAttributeExpression.h | 6 ++-- src/common/expression/LogicalExpression.cpp | 6 ++-- src/common/expression/LogicalExpression.h | 2 +- .../expression/MatchPathPatternExpression.h | 4 +-- src/common/expression/PropertyExpression.cpp | 2 +- .../expression/RelationalExpression.cpp | 6 ++-- .../expression/TextSearchExpression.cpp | 4 +-- src/common/expression/TextSearchExpression.h | 8 ++--- src/common/expression/UnaryExpression.cpp | 6 ++-- src/common/expression/VariableExpression.h | 4 +-- src/common/function/FunctionManager.cpp | 8 ++--- src/common/geo/GeoIndex.cpp | 2 +- src/common/geo/GeoUtils.h | 2 +- src/common/geo/io/wkb/WKBReader.cpp | 2 +- src/common/geo/io/wkb/WKBWriter.cpp | 2 +- src/common/id/Snowflake.cpp | 2 +- src/common/meta/GflagsManager.cpp | 2 +- src/common/meta/NebulaSchemaProvider.cpp | 2 +- src/common/stats/StatsManager-inl.h | 2 +- src/common/stats/StatsManager.h | 4 +-- src/common/utils/DefaultValueContext.h | 30 +++++++++---------- src/common/utils/IndexKeyUtils.cpp | 2 +- src/common/utils/IndexKeyUtils.h | 4 +-- src/graph/context/Iterator.cpp | 2 +- src/graph/context/Iterator.h | 18 +++++------ src/graph/executor/Executor.cpp | 2 +- .../executor/admin/SubmitJobExecutor.cpp | 2 +- .../executor/query/DataCollectExecutor.cpp | 2 +- .../optimizer/rule/PushEFilterDownRule.cpp | 2 +- .../optimizer/rule/PushFilterDownNodeRule.cpp | 6 ++-- .../rule/UnionAllIndexScanBaseRule.cpp | 2 +- src/graph/planner/match/SegmentsConnector.cpp | 2 +- src/graph/planner/plan/PlanNode.cpp | 2 +- src/graph/planner/plan/PlanNode.h | 8 ++--- src/graph/util/ParserUtil.cpp | 2 +- src/graph/util/ToJson.cpp | 2 +- src/graph/validator/LookupValidator.cpp | 2 +- src/graph/validator/Validator.cpp | 4 +-- src/graph/visitor/DeduceTypeVisitor.cpp | 6 ++-- src/graph/visitor/ExtractPropExprVisitor.cpp | 6 ++-- src/meta/processors/schema/SchemaUtil.cpp | 2 +- src/parser/AdminSentences.cpp | 26 ++++++++-------- src/parser/MaintainSentences.cpp | 8 ++--- src/parser/TraverseSentences.cpp | 4 +-- src/storage/GraphStorageServiceHandler.cpp | 2 +- src/storage/StorageServer.cpp | 6 ++-- .../context/StorageExpressionContext.h | 8 ++--- src/storage/exec/IndexExprContext.h | 4 +-- src/storage/exec/MultiTagNode.h | 6 ++-- src/storage/exec/QueryUtils.h | 4 +-- src/storage/exec/StorageIterator.h | 2 +- .../transaction/ChainProcessorFactory.cpp | 8 ++--- src/storage/transaction/ConsistUtil.cpp | 2 +- 63 files changed, 164 insertions(+), 166 deletions(-) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 16482bdfb51..7446ab95f12 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -2571,7 +2571,7 @@ folly::Future> MetaClient::heartbeat() { if (FileBasedClusterIdMan::persistInFile(resp.get_cluster_id(), FLAGS_cluster_id_path)) { options_.clusterId_.store(resp.get_cluster_id()); } else { - LOG(DFATAL) << "Can't persist the clusterId in file " << FLAGS_cluster_id_path; + DLOG(FATAL) << "Can't persist the clusterId in file " << FLAGS_cluster_id_path; return false; } } diff --git a/src/codec/RowReaderV1.cpp b/src/codec/RowReaderV1.cpp index 0d17b3aef76..a8f9023c258 100644 --- a/src/codec/RowReaderV1.cpp +++ b/src/codec/RowReaderV1.cpp @@ -142,8 +142,7 @@ int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept { break; } default: { - // TODO - LOG(DFATAL) << "Unimplemented"; + LOG(FATAL) << "Unimplemented"; return -1; } } diff --git a/src/codec/RowReaderV2.cpp b/src/codec/RowReaderV2.cpp index cdfe207f1e7..384a171494f 100644 --- a/src/codec/RowReaderV2.cpp +++ b/src/codec/RowReaderV2.cpp @@ -208,8 +208,7 @@ Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept { case PropertyType::UNKNOWN: break; } - LOG(DFATAL) << "Should not reach here, illegal property type: " - << static_cast(field->type()); + LOG(FATAL) << "Should not reach here, illegal property type: " << static_cast(field->type()); return Value::kNullBadType; } diff --git a/src/codec/RowReaderWrapper.cpp b/src/codec/RowReaderWrapper.cpp index 77abcf81323..5a100fabcfb 100644 --- a/src/codec/RowReaderWrapper.cpp +++ b/src/codec/RowReaderWrapper.cpp @@ -89,7 +89,7 @@ RowReaderWrapper::RowReaderWrapper(const meta::SchemaProviderIf* schema, readerV2_.resetImpl(schema, row); currReader_ = &readerV2_; } else { - LOG(DFATAL) << "Should not reach here"; + LOG(FATAL) << "Should not reach here"; readerV2_.resetImpl(schema, row); currReader_ = &readerV2_; } diff --git a/src/codec/RowWriterV2.cpp b/src/codec/RowWriterV2.cpp index c6466191f55..17bc7c3a6d4 100644 --- a/src/codec/RowWriterV2.cpp +++ b/src/codec/RowWriterV2.cpp @@ -63,7 +63,7 @@ RowWriterV2::RowWriterV2(const meta::SchemaProviderIf* schema) header = 0x0F; // 0x08 | 0x07, seven bytes for the schema version headerLen_ = 8; } else { - LOG(DFATAL) << "Schema version too big"; + LOG(FATAL) << "Schema version too big"; header = 0x0F; // 0x08 | 0x07, seven bytes for the schema version headerLen_ = 8; } @@ -139,7 +139,7 @@ RowWriterV2::RowWriterV2(RowReader& reader) : RowWriterV2(reader.getSchema()) { set(i, v.moveDuration()); break; default: - LOG(DFATAL) << "Invalid data: " << v << ", type: " << v.typeName(); + LOG(FATAL) << "Invalid data: " << v << ", type: " << v.typeName(); isSet_[i] = false; continue; } @@ -856,9 +856,9 @@ WriteResult RowWriterV2::checkUnsetFields() noexcept { r = write(i, defVal.getDuration()); break; default: - LOG(DFATAL) << "Unsupported default value type: " << defVal.typeName() - << ", default value: " << defVal - << ", default value expr: " << field->defaultValue(); + LOG(FATAL) << "Unsupported default value type: " << defVal.typeName() + << ", default value: " << defVal + << ", default value expr: " << field->defaultValue(); return WriteResult::TYPE_MISMATCH; } } else { diff --git a/src/common/base/Arena.cpp b/src/common/base/Arena.cpp index ff2f49e6806..0775438bdfe 100644 --- a/src/common/base/Arena.cpp +++ b/src/common/base/Arena.cpp @@ -16,7 +16,7 @@ void* Arena::allocateAligned(const std::size_t alloc) { kAlignment - (reinterpret_cast(currentPtr_) & (kAlignment - 1)); const std::size_t consumption = alloc + pad; if (UNLIKELY(consumption > kMaxChunkSize)) { - LOG(DFATAL) << "Arena can't allocate so large memory."; + DLOG(FATAL) << "Arena can't allocate so large memory."; return nullptr; } if (LIKELY(consumption <= availableSize_)) { diff --git a/src/common/base/Status.cpp b/src/common/base/Status.cpp index 3954b113e94..d80c31cf370 100644 --- a/src/common/base/Status.cpp +++ b/src/common/base/Status.cpp @@ -100,7 +100,7 @@ const char *Status::toString(Code code) { case kSessionNotFound: return "SessionNotFound"; } - LOG(DFATAL) << "Invalid status code: " << static_cast(code); + DLOG(FATAL) << "Invalid status code: " << static_cast(code); return ""; } diff --git a/src/common/datatypes/Value.cpp b/src/common/datatypes/Value.cpp index 583c1c1575f..b0cd600effe 100644 --- a/src/common/datatypes/Value.cpp +++ b/src/common/datatypes/Value.cpp @@ -1507,7 +1507,7 @@ folly::dynamic Value::toJson() const { // no default so the compiler will warning when lack } - LOG(DFATAL) << "Unknown value type " << static_cast(type_); + DLOG(FATAL) << "Unknown value type " << static_cast(type_); return folly::dynamic(nullptr); } @@ -1553,7 +1553,7 @@ folly::dynamic Value::getMetaData() const { break; } - LOG(DFATAL) << "Unknown value type " << static_cast(type_); + DLOG(FATAL) << "Unknown value type " << static_cast(type_); return folly::dynamic(nullptr); } @@ -1581,7 +1581,7 @@ std::string Value::toString() const { case NullType::OUT_OF_RANGE: return "__NULL_OUT_OF_RANGE__"; } - LOG(DFATAL) << "Unknown Null type " << static_cast(getNull()); + DLOG(FATAL) << "Unknown Null type " << static_cast(getNull()); return "__NULL_BAD_TYPE__"; } case Value::Type::BOOL: { @@ -1635,7 +1635,7 @@ std::string Value::toString() const { // no default so the compiler will warning when lack } - LOG(DFATAL) << "Unknown value type " << static_cast(type_); + DLOG(FATAL) << "Unknown value type " << static_cast(type_); return "__NULL_BAD_TYPE__"; } @@ -1858,7 +1858,7 @@ Value Value::lessThan(const Value& v) const { return kNullBadType; } } - LOG(DFATAL) << "Unknown type " << static_cast(v.type()); + DLOG(FATAL) << "Unknown type " << static_cast(v.type()); return Value::kNullBadType; } @@ -1957,7 +1957,7 @@ Value Value::equal(const Value& v) const { return false; } } - LOG(DFATAL) << "Unknown type " << static_cast(v.type()); + DLOG(FATAL) << "Unknown type " << static_cast(v.type()); return Value::kNullBadType; } @@ -1979,7 +1979,7 @@ bool Value::implicitBool() const { case Type::LIST: return !getList().empty(); default: - LOG(DFATAL) << "Impossible to reach here!"; + DLOG(FATAL) << "Impossible to reach here!"; return false; } } @@ -2262,7 +2262,7 @@ Value operator+(const Value& lhs, const Value& rhs) { return Value::kNullValue; } } - LOG(DFATAL) << "Unknown type: " << rhs.type(); + DLOG(FATAL) << "Unknown type: " << rhs.type(); return Value::kNullBadType; } case Value::Type::VERTEX: { @@ -2696,7 +2696,7 @@ bool operator<(const Value& lhs, const Value& rhs) { return lhs.getGeography() < rhs.getGeography(); } case Value::Type::DURATION: { - LOG(DFATAL) << "Duration is not comparable."; + DLOG(FATAL) << "Duration is not comparable."; return false; } case Value::Type::NULLVALUE: @@ -2704,7 +2704,7 @@ bool operator<(const Value& lhs, const Value& rhs) { return false; } } - LOG(DFATAL) << "Unknown type " << static_cast(lType); + DLOG(FATAL) << "Unknown type " << static_cast(lType); return false; } @@ -2800,7 +2800,7 @@ bool Value::equals(const Value& rhs) const { return false; } } - LOG(DFATAL) << "Unknown type " << static_cast(type()); + DLOG(FATAL) << "Unknown type " << static_cast(type()); return false; } @@ -2858,11 +2858,11 @@ std::size_t Value::hash() const { return std::hash()(getDuration()); } case Type::DATASET: { - LOG(DFATAL) << "Hash for DATASET has not been implemented"; + DLOG(FATAL) << "Hash for DATASET has not been implemented"; break; } default: { - LOG(DFATAL) << "Unknown type"; + DLOG(FATAL) << "Unknown type"; } } return ~0UL; diff --git a/src/common/expression/ArithmeticExpression.cpp b/src/common/expression/ArithmeticExpression.cpp index 492d63cf3a8..0a4c0b52b2e 100644 --- a/src/common/expression/ArithmeticExpression.cpp +++ b/src/common/expression/ArithmeticExpression.cpp @@ -30,7 +30,7 @@ const Value& ArithmeticExpression::eval(ExpressionContext& ctx) { result_ = lhs % rhs; break; default: - LOG(DFATAL) << "Unknown type: " << kind_; + DLOG(FATAL) << "Unknown type: " << kind_; result_ = Value::kNullBadType; } return result_; @@ -55,8 +55,8 @@ std::string ArithmeticExpression::toString() const { op = "%"; break; default: - LOG(DFATAL) << "Illegal kind for arithmetic expression: " << static_cast(kind()); - op = "illegal symbol "; + DLOG(FATAL) << "Illegal kind for arithmetic expression: " << static_cast(kind()); + op = " Invalid arithmetic expression "; } std::stringstream out; out << "(" << (lhs_ ? lhs_->toString() : "") << op << (rhs_ ? rhs_->toString() : "") << ")"; diff --git a/src/common/expression/Expression.cpp b/src/common/expression/Expression.cpp index 2a6ef5dbfb1..ec36dab0b67 100644 --- a/src/common/expression/Expression.cpp +++ b/src/common/expression/Expression.cpp @@ -395,7 +395,7 @@ Expression* Expression::decode(ObjectPool* pool, Expression::Decoder& decoder) { return exp; } case Expression::Kind::kInputProperty: { - LOG(DFATAL) << "Should not decode input property expression"; + LOG(FATAL) << "Should not decode input property expression"; return exp; } case Expression::Kind::kVarProperty: { @@ -459,7 +459,7 @@ Expression* Expression::decode(ObjectPool* pool, Expression::Decoder& decoder) { return exp; } case Expression::Kind::kVersionedVar: { - LOG(DFATAL) << "Should not decode version variable expression"; + LOG(FATAL) << "Should not decode version variable expression"; return exp; } case Expression::Kind::kUUID: { @@ -516,17 +516,17 @@ Expression* Expression::decode(ObjectPool* pool, Expression::Decoder& decoder) { case Expression::Kind::kTSWildcard: case Expression::Kind::kTSRegexp: case Expression::Kind::kTSFuzzy: { - LOG(DFATAL) << "Should not decode text search expression"; + LOG(FATAL) << "Should not decode text search expression"; return exp; } case Expression::Kind::kMatchPathPattern: { - LOG(DFATAL) << "Should not decode match path pattern expression."; + LOG(FATAL) << "Should not decode match path pattern expression."; return exp; } // no default so the compiler will warning when lack } - LOG(DFATAL) << "Unknown expression: " << decoder.getHexStr(); + LOG(FATAL) << "Unknown expression: " << decoder.getHexStr(); return exp; } diff --git a/src/common/expression/LabelAttributeExpression.h b/src/common/expression/LabelAttributeExpression.h index 43105f0c04f..40cee0d7fb0 100644 --- a/src/common/expression/LabelAttributeExpression.h +++ b/src/common/expression/LabelAttributeExpression.h @@ -34,7 +34,7 @@ class LabelAttributeExpression final : public Expression { } const Value& eval(ExpressionContext&) override { - DLOG(DFATAL) << "LabelAttributeExpression has to be rewritten"; + DLOG(FATAL) << "LabelAttributeExpression has to be rewritten"; return Value::kNullBadData; } @@ -76,11 +76,11 @@ class LabelAttributeExpression final : public Expression { } void writeTo(Encoder&) const override { - LOG(DFATAL) << "LabelAttributeExpression not supported to encode."; + LOG(FATAL) << "LabelAttributeExpression not supported to encode."; } void resetFrom(Decoder&) override { - LOG(DFATAL) << "LabelAttributeExpression not supported to decode."; + LOG(FATAL) << "LabelAttributeExpression not supported to decode."; } private: diff --git a/src/common/expression/LogicalExpression.cpp b/src/common/expression/LogicalExpression.cpp index a157f040347..bec11fac8e5 100644 --- a/src/common/expression/LogicalExpression.cpp +++ b/src/common/expression/LogicalExpression.cpp @@ -19,7 +19,7 @@ const Value &LogicalExpression::eval(ExpressionContext &ctx) { case Kind::kLogicalXor: return evalXor(ctx); default: - LOG(DFATAL) << "Illegal kind for logical expression: " << static_cast(kind()); + DLOG(FATAL) << "Illegal kind for logical expression: " << static_cast(kind()); return Value::kNullBadType; } } @@ -116,8 +116,8 @@ std::string LogicalExpression::toString() const { op = " XOR "; break; default: - LOG(DFATAL) << "Illegal kind for logical expression: " << static_cast(kind()); - op = " illegal symbol "; + DLOG(FATAL) << "Illegal kind for logical expression: " << static_cast(kind()); + op = " Invalid logical expression "; } std::string buf; buf.reserve(256); diff --git a/src/common/expression/LogicalExpression.h b/src/common/expression/LogicalExpression.h index c1aa889b8e4..22cf72143e9 100644 --- a/src/common/expression/LogicalExpression.h +++ b/src/common/expression/LogicalExpression.h @@ -99,7 +99,7 @@ class LogicalExpression final : public Expression { } else if (kind_ == Kind::kLogicalOr) { kind_ = Kind::kLogicalAnd; } else { - LOG(DFATAL) << "Should not reverse logical expression except and/or kind."; + DLOG(FATAL) << "Should not reverse logical expression except and/or kind."; } } diff --git a/src/common/expression/MatchPathPatternExpression.h b/src/common/expression/MatchPathPatternExpression.h index c4547a51d76..1438ddcc293 100644 --- a/src/common/expression/MatchPathPatternExpression.h +++ b/src/common/expression/MatchPathPatternExpression.h @@ -60,12 +60,12 @@ class MatchPathPatternExpression final : public Expression { // This expression contains variable implicitly, so we don't support persist or transform it. void writeTo(Encoder&) const override { - LOG(DFATAL) << "Not implemented"; + LOG(FATAL) << "Not implemented"; } // This expression contains variable implicitly, so we don't support persist or transform it. void resetFrom(Decoder&) override { - LOG(DFATAL) << "Not implemented"; + LOG(FATAL) << "Not implemented"; } private: diff --git a/src/common/expression/PropertyExpression.cpp b/src/common/expression/PropertyExpression.cpp index 81afd944433..09ec28eb4a6 100644 --- a/src/common/expression/PropertyExpression.cpp +++ b/src/common/expression/PropertyExpression.cpp @@ -46,7 +46,7 @@ void PropertyExpression::resetFrom(Decoder& decoder) { const Value& PropertyExpression::eval(ExpressionContext& ctx) { // TODO maybe cypher need it. UNUSED(ctx); - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Value::kNullBadType; } diff --git a/src/common/expression/RelationalExpression.cpp b/src/common/expression/RelationalExpression.cpp index 0c328f72d86..887562f1cb3 100644 --- a/src/common/expression/RelationalExpression.cpp +++ b/src/common/expression/RelationalExpression.cpp @@ -195,7 +195,7 @@ const Value& RelationalExpression::eval(ExpressionContext& ctx) { break; } default: - LOG(DFATAL) << "Unknown type: " << kind_; + DLOG(FATAL) << "Unknown type: " << kind_; result_ = Value::kNullBadType; } return result_; @@ -250,8 +250,8 @@ std::string RelationalExpression::toString() const { op = " NOT ENDS WITH "; break; default: - LOG(DFATAL) << "Illegal kind for relational expression: " << static_cast(kind()); - op = " illegal symbol "; + DLOG(FATAL) << "Illegal kind for relational expression: " << static_cast(kind()); + op = " Invalid relational expression "; } std::stringstream out; out << "(" << (lhs_ ? lhs_->toString() : "") << op << (rhs_ ? rhs_->toString() : "") << ")"; diff --git a/src/common/expression/TextSearchExpression.cpp b/src/common/expression/TextSearchExpression.cpp index f10557cc4b2..a8e556def24 100644 --- a/src/common/expression/TextSearchExpression.cpp +++ b/src/common/expression/TextSearchExpression.cpp @@ -66,8 +66,8 @@ std::string TextSearchExpression::toString() const { break; } default: { - LOG(DFATAL) << "Illegal kind for text search expression: " << static_cast(kind()); - buf = "illegal symbol("; + DLOG(FATAL) << "Illegal kind for text search expression: " << static_cast(kind()); + buf = "invalid text search expression("; } } buf += arg_ ? arg_->toString() : ""; diff --git a/src/common/expression/TextSearchExpression.h b/src/common/expression/TextSearchExpression.h index 281c8a15bb6..078068111ae 100644 --- a/src/common/expression/TextSearchExpression.h +++ b/src/common/expression/TextSearchExpression.h @@ -114,12 +114,12 @@ class TextSearchExpression : public Expression { bool operator==(const Expression& rhs) const override; const Value& eval(ExpressionContext&) override { - LOG(DFATAL) << "TextSearchExpression has to be rewritten"; + DLOG(FATAL) << "TextSearchExpression has to be rewritten"; return Value::kNullBadData; } void accept(ExprVisitor*) override { - LOG(DFATAL) << "TextSearchExpression has to be rewritten"; + DLOG(FATAL) << "TextSearchExpression has to be rewritten"; } std::string toString() const override; @@ -149,11 +149,11 @@ class TextSearchExpression : public Expression { } void writeTo(Encoder&) const override { - LOG(DFATAL) << "TextSearchExpression has to be rewritten"; + LOG(FATAL) << "TextSearchExpression has to be rewritten"; } void resetFrom(Decoder&) override { - LOG(DFATAL) << "TextSearchExpression has to be reset"; + LOG(FATAL) << "TextSearchExpression has to be reset"; } private: diff --git a/src/common/expression/UnaryExpression.cpp b/src/common/expression/UnaryExpression.cpp index 75223036d05..d230111ad0d 100644 --- a/src/common/expression/UnaryExpression.cpp +++ b/src/common/expression/UnaryExpression.cpp @@ -87,7 +87,7 @@ const Value& UnaryExpression::eval(ExpressionContext& ctx) { break; } default: { - LOG(DFATAL) << "Unknown type: " << kind_; + DLOG(FATAL) << "Unknown type: " << kind_; result_ = Value::kNullBadType; } } @@ -121,8 +121,8 @@ std::string UnaryExpression::toString() const { case Kind::kIsNotEmpty: return (operand_ ? operand_->toString() : "") + " IS NOT EMPTY"; default: - LOG(DFATAL) << "Illegal kind for unary expression: " << static_cast(kind()); - op = "illegal symbol "; + DLOG(FATAL) << "Illegal kind for unary expression: " << static_cast(kind()); + op = " Invalid unary expression "; } std::stringstream out; out << op << "(" << (operand_ ? operand_->toString() : "") << ")"; diff --git a/src/common/expression/VariableExpression.h b/src/common/expression/VariableExpression.h index 5c73000d15c..52e6a3907f0 100644 --- a/src/common/expression/VariableExpression.h +++ b/src/common/expression/VariableExpression.h @@ -107,11 +107,11 @@ class VersionedVariableExpression final : public Expression { : Expression(pool, Kind::kVersionedVar), var_(var), version_(version) {} void writeTo(Encoder&) const override { - LOG(DFATAL) << "VersionedVariableExpression not support to encode."; + LOG(FATAL) << "VersionedVariableExpression not support to encode."; } void resetFrom(Decoder&) override { - LOG(DFATAL) << "VersionedVariableExpression not support to decode."; + LOG(FATAL) << "VersionedVariableExpression not support to decode."; } private: diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index d985a3a8079..74a762a93d9 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -1692,7 +1692,7 @@ FunctionManager::FunctionManager() { } } default: - LOG(DFATAL) << "Unexpected arguments count " << args.size(); + DLOG(FATAL) << "Unexpected arguments count " << args.size(); return Value::kNullBadType; } }; @@ -1733,7 +1733,7 @@ FunctionManager::FunctionManager() { } } default: - LOG(DFATAL) << "Unexpected arguments count " << args.size(); + DLOG(FATAL) << "Unexpected arguments count " << args.size(); return Value::kNullBadType; } }; @@ -1775,7 +1775,7 @@ FunctionManager::FunctionManager() { } } default: - LOG(DFATAL) << "Unexpected arguments count " << args.size(); + DLOG(FATAL) << "Unexpected arguments count " << args.size(); return Value::kNullBadType; } }; @@ -2793,7 +2793,7 @@ FunctionManager::FunctionManager() { const auto &p = args[0].get().getPath(); const std::size_t nodeIndex = args[1].get().getInt(); if (nodeIndex < 0 || nodeIndex >= (1 + p.steps.size())) { - LOG(DFATAL) << "Out of range node index."; + DLOG(FATAL) << "Out of range node index."; return Value::kNullOutOfRange; } if (nodeIndex == 0) { diff --git a/src/common/geo/GeoIndex.cpp b/src/common/geo/GeoIndex.cpp index 67080c169fd..0995afc99eb 100644 --- a/src/common/geo/GeoIndex.cpp +++ b/src/common/geo/GeoIndex.cpp @@ -118,7 +118,7 @@ std::vector GeoIndex::dWithin(const Geography& g, double distance) co return intersects(gBuffer); } default: - LOG(DFATAL) + DLOG(FATAL) << "Geography shapes other than Point/LineString/Polygon are not currently supported"; return {}; } diff --git a/src/common/geo/GeoUtils.h b/src/common/geo/GeoUtils.h index f140fe10138..cf4e9cf3863 100644 --- a/src/common/geo/GeoUtils.h +++ b/src/common/geo/GeoUtils.h @@ -47,7 +47,7 @@ class GeoUtils final { return std::make_unique(std::move(s2Loops), S2Debug::DISABLE); } default: - LOG(DFATAL) + DLOG(FATAL) << "Geography shapes other than Point/LineString/Polygon are not currently supported"; return nullptr; } diff --git a/src/common/geo/io/wkb/WKBReader.cpp b/src/common/geo/io/wkb/WKBReader.cpp index 434c0d13c0e..1a9c092f179 100644 --- a/src/common/geo/io/wkb/WKBReader.cpp +++ b/src/common/geo/io/wkb/WKBReader.cpp @@ -31,7 +31,7 @@ StatusOr WKBReader::read(const std::string &wkb) { return readPolygon(); } default: - LOG(DFATAL) + DLOG(FATAL) << "Geography shapes other than Point/LineString/Polygon are not currently supported"; return Status::Error( "Geography shapes other than Point/LineString/Polygon are not currently supported"); diff --git a/src/common/geo/io/wkb/WKBWriter.cpp b/src/common/geo/io/wkb/WKBWriter.cpp index cac1d44ec2e..7873f06fbc1 100644 --- a/src/common/geo/io/wkb/WKBWriter.cpp +++ b/src/common/geo/io/wkb/WKBWriter.cpp @@ -32,7 +32,7 @@ std::string WKBWriter::write(const Geography& geog, ByteOrder byteOrder) { return os_.str(); } default: - LOG(DFATAL) + DLOG(FATAL) << "Geometry shapes other than Point/LineString/Polygon are not currently supported"; return ""; } diff --git a/src/common/id/Snowflake.cpp b/src/common/id/Snowflake.cpp index 4aacedb5b43..4701c9b0d0e 100644 --- a/src/common/id/Snowflake.cpp +++ b/src/common/id/Snowflake.cpp @@ -10,7 +10,7 @@ bool Snowflake::initWorkerId(meta::MetaClient* client) { const std::string& ip = client->getLocalIp(); auto result = client->getWorkerId(ip).get(); if (!result.ok()) { - LOG(DFATAL) << "Failed to get workerId from meta server"; + DLOG(FATAL) << "Failed to get workerId from meta server"; return false; } workerId_ = result.value(); diff --git a/src/common/meta/GflagsManager.cpp b/src/common/meta/GflagsManager.cpp index a25d2708fef..b1e100aabf0 100644 --- a/src/common/meta/GflagsManager.cpp +++ b/src/common/meta/GflagsManager.cpp @@ -209,7 +209,7 @@ std::string GflagsManager::ValueToGflagString(const Value& val) { return os.str(); } default: { - LOG(DFATAL) << "Unsupported type for gflags"; + DLOG(FATAL) << "Unsupported type for gflags"; return ""; } } diff --git a/src/common/meta/NebulaSchemaProvider.cpp b/src/common/meta/NebulaSchemaProvider.cpp index 5cb9a891fdf..51bc08c7de5 100644 --- a/src/common/meta/NebulaSchemaProvider.cpp +++ b/src/common/meta/NebulaSchemaProvider.cpp @@ -176,7 +176,7 @@ std::size_t NebulaSchemaProvider::fieldSize(PropertyType type, std::size_t fixed case PropertyType::UNKNOWN: break; } - LOG(DFATAL) << "Incorrect field type " << static_cast(type); + DLOG(FATAL) << "Incorrect field type " << static_cast(type); return 0; } diff --git a/src/common/stats/StatsManager-inl.h b/src/common/stats/StatsManager-inl.h index 0415735e2f4..e5725152958 100644 --- a/src/common/stats/StatsManager-inl.h +++ b/src/common/stats/StatsManager-inl.h @@ -26,7 +26,7 @@ StatsManager::VT StatsManager::readValue(StatsHolder& stats, return stats.template rate(level); } - LOG(DFATAL) << "Unknown statistic method"; + DLOG(FATAL) << "Unknown statistic method"; return StatsManager::VT(0); } diff --git a/src/common/stats/StatsManager.h b/src/common/stats/StatsManager.h index b6bd4f038a9..07177651328 100644 --- a/src/common/stats/StatsManager.h +++ b/src/common/stats/StatsManager.h @@ -29,11 +29,11 @@ class CounterId final { CounterId& operator=(const CounterId& right) { if (!right.valid()) { - LOG(FATAL) << "Invalid counter id"; + DLOG(FATAL) << "Invalid counter id"; } if (valid()) { // Already assigned - LOG(DFATAL) << "CounterId cannot be assigned twice"; + DLOG(FATAL) << "CounterId cannot be assigned twice"; } index_ = right.index_; isHisto_ = right.isHisto_; diff --git a/src/common/utils/DefaultValueContext.h b/src/common/utils/DefaultValueContext.h index 89a5aad4d9b..e72278858b5 100644 --- a/src/common/utils/DefaultValueContext.h +++ b/src/common/utils/DefaultValueContext.h @@ -14,66 +14,66 @@ namespace nebula { class DefaultValueContext final : public ExpressionContext { public: const Value& getVar(const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } void setInnerVar(const std::string&, Value) override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; } const Value& getInnerVar(const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } const Value& getVersionedVar(const std::string&, int64_t) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } const Value& getVarProp(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } StatusOr getVarPropIndex(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Status::Error("Unimplemented"); } Value getEdgeProp(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } Value getTagProp(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } Value getSrcProp(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } const Value& getDstProp(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } const Value& getInputProp(const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } StatusOr getInputPropIndex(const std::string&) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Status::Error("Unimplemented"); } const Value& getColumn(int32_t) const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } @@ -81,12 +81,12 @@ class DefaultValueContext final : public ExpressionContext { Value getVertex(const std::string& name = "") const override { UNUSED(name); - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } Value getEdge() const override { - LOG(DFATAL) << "Not allowed to call"; + DLOG(FATAL) << "Not allowed to call"; return Value::kEmpty; } }; diff --git a/src/common/utils/IndexKeyUtils.cpp b/src/common/utils/IndexKeyUtils.cpp index 912d4125d25..9b0fa4eda31 100644 --- a/src/common/utils/IndexKeyUtils.cpp +++ b/src/common/utils/IndexKeyUtils.cpp @@ -258,7 +258,7 @@ Status IndexKeyUtils::checkValue(const Value& v, bool isNullable) { return Status::Error("Out of range"); } } - LOG(DFATAL) << "Unknown Null type " << static_cast(v.getNull()); + DLOG(FATAL) << "Unknown Null type " << static_cast(v.getNull()); return Status::Error("Unknown Null type"); } diff --git a/src/common/utils/IndexKeyUtils.h b/src/common/utils/IndexKeyUtils.h index 977800a20fc..8a8b1753c20 100644 --- a/src/common/utils/IndexKeyUtils.h +++ b/src/common/utils/IndexKeyUtils.h @@ -142,11 +142,11 @@ class IndexKeyUtils final { return encodeDateTime(v.getDateTime()); } case Value::Type::GEOGRAPHY: { - LOG(DFATAL) << "Should call encodeGeography separately"; + DLOG(FATAL) << "Should call encodeGeography separately"; return ""; } default: - LOG(DFATAL) << "Unsupported default value type"; + DLOG(FATAL) << "Unsupported default value type"; } return ""; } diff --git a/src/graph/context/Iterator.cpp b/src/graph/context/Iterator.cpp index 187a73ee81a..5b2b94ffac1 100644 --- a/src/graph/context/Iterator.cpp +++ b/src/graph/context/Iterator.cpp @@ -758,7 +758,7 @@ PropIter::PropIter(std::shared_ptr value, bool checkMemory) auto& ds = value->getDataSet(); auto status = makeDataSetIndex(ds); if (UNLIKELY(!status.ok())) { - LOG(DFATAL) << status; + DLOG(FATAL) << status; clear(); return; } diff --git a/src/graph/context/Iterator.h b/src/graph/context/Iterator.h index 77441e289e2..1b4119f0a32 100644 --- a/src/graph/context/Iterator.h +++ b/src/graph/context/Iterator.h @@ -136,12 +136,12 @@ class Iterator { } virtual const Value& getTagProp(const std::string&, const std::string&) const { - LOG(DFATAL) << "Shouldn't call the unimplemented method"; + DLOG(FATAL) << "Shouldn't call the unimplemented method"; return Value::kEmpty; } virtual const Value& getEdgeProp(const std::string&, const std::string&) const { - LOG(DFATAL) << "Shouldn't call the unimplemented method"; + DLOG(FATAL) << "Shouldn't call the unimplemented method"; return Value::kEmpty; } @@ -203,11 +203,11 @@ class DefaultIter final : public Iterator { } void select(std::size_t, std::size_t) override { - LOG(DFATAL) << "Unimplemented method for default iterator."; + DLOG(FATAL) << "Unimplemented method for default iterator."; } void sample(int64_t) override { - LOG(DFATAL) << "Unimplemented default iterator."; + DLOG(FATAL) << "Unimplemented default iterator."; } void clear() override { @@ -219,27 +219,27 @@ class DefaultIter final : public Iterator { } const Value& getColumn(const std::string& /* col */) const override { - LOG(DFATAL) << "This method should not be invoked"; + DLOG(FATAL) << "This method should not be invoked"; return Value::kEmpty; } const Value& getColumn(int32_t) const override { - LOG(DFATAL) << "This method should not be invoked"; + DLOG(FATAL) << "This method should not be invoked"; return Value::kEmpty; } StatusOr getColumnIndex(const std::string&) const override { - LOG(DFATAL) << "This method should not be invoked"; + DLOG(FATAL) << "This method should not be invoked"; return Status::Error("Unimplemented method"); } const Row* row() const override { - LOG(DFATAL) << "This method should not be invoked"; + DLOG(FATAL) << "This method should not be invoked"; return nullptr; } Row moveRow() override { - LOG(DFATAL) << "This method should not be invoked"; + DLOG(FATAL) << "This method should not be invoked"; return Row{}; } diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 64bd3064b18..e3d62265b8b 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -566,7 +566,7 @@ Executor *Executor::makeExecutor(QueryContext *qctx, const PlanNode *node) { return pool->makeAndAdd(node, qctx); } case PlanNode::Kind::kUnknown: { - LOG(DFATAL) << "Unknown plan node kind " << static_cast(node->kind()); + DLOG(FATAL) << "Unknown plan node kind " << static_cast(node->kind()); break; } } diff --git a/src/graph/executor/admin/SubmitJobExecutor.cpp b/src/graph/executor/admin/SubmitJobExecutor.cpp index 6d41c9913f1..2af4b6cc741 100644 --- a/src/graph/executor/admin/SubmitJobExecutor.cpp +++ b/src/graph/executor/admin/SubmitJobExecutor.cpp @@ -101,7 +101,7 @@ StatusOr SubmitJobExecutor::buildResult(meta::cpp2::JobOp jobOp, } // no default so the compiler will warning when lack } - LOG(DFATAL) << "Unknown job operation " << static_cast(jobOp); + DLOG(FATAL) << "Unknown job operation " << static_cast(jobOp); return Status::Error("Unknown job job operation %d.", static_cast(jobOp)); } diff --git a/src/graph/executor/query/DataCollectExecutor.cpp b/src/graph/executor/query/DataCollectExecutor.cpp index 9281e05d33b..7e15bf23459 100644 --- a/src/graph/executor/query/DataCollectExecutor.cpp +++ b/src/graph/executor/query/DataCollectExecutor.cpp @@ -50,7 +50,7 @@ folly::Future DataCollectExecutor::doCollect() { break; } default: - LOG(DFATAL) << "Unknown data collect type: " << static_cast(dc->kind()); + DLOG(FATAL) << "Unknown data collect type: " << static_cast(dc->kind()); return Status::Error("Unknown data collect type: %d.", static_cast(dc->kind())); } ResultBuilder builder; diff --git a/src/graph/optimizer/rule/PushEFilterDownRule.cpp b/src/graph/optimizer/rule/PushEFilterDownRule.cpp index f18ae8798eb..ac886530210 100644 --- a/src/graph/optimizer/rule/PushEFilterDownRule.cpp +++ b/src/graph/optimizer/rule/PushEFilterDownRule.cpp @@ -174,7 +174,7 @@ std::string PushEFilterDownRule::toString() const { ret = EdgePropertyExpression::make(pool, std::move(edgeNameResult).value(), exp->prop()); break; default: - LOG(DFATAL) << "Unexpected expr: " << exp->kind(); + DLOG(FATAL) << "Unexpected expr: " << exp->kind(); } return ret; } diff --git a/src/graph/optimizer/rule/PushFilterDownNodeRule.cpp b/src/graph/optimizer/rule/PushFilterDownNodeRule.cpp index 20fcfb4a7d2..9c89cd16c63 100644 --- a/src/graph/optimizer/rule/PushFilterDownNodeRule.cpp +++ b/src/graph/optimizer/rule/PushFilterDownNodeRule.cpp @@ -55,7 +55,7 @@ StatusOr PushFilterDownNodeRule::transform( auto *append = static_cast(node); vFilter = append->vFilter()->clone(); } else { - LOG(DFATAL) << "Unsupported node kind: " << node->kind(); + DLOG(FATAL) << "Unsupported node kind: " << node->kind(); return TransformResult::noTransform(); } auto visitor = graph::ExtractFilterExprVisitor::makePushGetVertices(pool); @@ -83,7 +83,7 @@ StatusOr PushFilterDownNodeRule::transform( append->setVertexFilter(remainedExpr); append->setFilter(vFilter); } else { - LOG(DFATAL) << "Unsupported node kind: " << newExplore->kind(); + DLOG(FATAL) << "Unsupported node kind: " << newExplore->kind(); return TransformResult::noTransform(); } @@ -111,7 +111,7 @@ bool PushFilterDownNodeRule::match(OptContext *octx, const MatchedResult &matche return false; } } else { - LOG(DFATAL) << "Unexpected node kind: " << node->kind(); + DLOG(FATAL) << "Unexpected node kind: " << node->kind(); return false; } return true; diff --git a/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp b/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp index 93928c8d68b..cc1bb6a41fd 100644 --- a/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp +++ b/src/graph/optimizer/rule/UnionAllIndexScanBaseRule.cpp @@ -167,7 +167,7 @@ StatusOr UnionAllIndexScanBaseRule::transform(OptContext* ctx, break; } default: - LOG(DFATAL) << "Invalid expression kind: " << static_cast(conditionType); + DLOG(FATAL) << "Invalid expression kind: " << static_cast(conditionType); return TransformResult::noTransform(); } diff --git a/src/graph/planner/match/SegmentsConnector.cpp b/src/graph/planner/match/SegmentsConnector.cpp index 2c6ecb73707..cc29fb85cb1 100644 --- a/src/graph/planner/match/SegmentsConnector.cpp +++ b/src/graph/planner/match/SegmentsConnector.cpp @@ -142,7 +142,7 @@ SubPlan SegmentsConnector::addInput(const SubPlan& left, const SubPlan& right, b siLeft->setLeftDep(const_cast(right.root)); siLeft->setLeftVar(right.root->outputVar()); } else { - LOG(DFATAL) << "Unsupported plan node: " << left.tail->kind(); + DLOG(FATAL) << "Unsupported plan node: " << left.tail->kind(); return newPlan; } newPlan.tail = right.tail; diff --git a/src/graph/planner/plan/PlanNode.cpp b/src/graph/planner/plan/PlanNode.cpp index 76f49d07a83..009812a5ab5 100644 --- a/src/graph/planner/plan/PlanNode.cpp +++ b/src/graph/planner/plan/PlanNode.cpp @@ -309,7 +309,7 @@ const char* PlanNode::toString(PlanNode::Kind kind) { return "GetDstBySrc"; // no default so the compiler will warning when lack } - LOG(DFATAL) << "Impossible kind plan node " << static_cast(kind); + DLOG(FATAL) << "Impossible kind plan node " << static_cast(kind); return "Unknown"; } diff --git a/src/graph/planner/plan/PlanNode.h b/src/graph/planner/plan/PlanNode.h index 4fad0c71074..aa4a8a78cd7 100644 --- a/src/graph/planner/plan/PlanNode.h +++ b/src/graph/planner/plan/PlanNode.h @@ -351,7 +351,7 @@ class SingleDependencyNode : public PlanNode { } PlanNode* clone() const override { - LOG(DFATAL) << "Shouldn't call the unimplemented method"; + DLOG(FATAL) << "Shouldn't call the unimplemented method"; return nullptr; } @@ -373,7 +373,7 @@ class SingleInputNode : public SingleDependencyNode { std::unique_ptr explain() const override; PlanNode* clone() const override { - LOG(DFATAL) << "Shouldn't call the unimplemented method"; + DLOG(FATAL) << "Shouldn't call the unimplemented method"; return nullptr; } @@ -423,7 +423,7 @@ class BinaryInputNode : public PlanNode { } PlanNode* clone() const override { - LOG(DFATAL) << "Shouldn't call the unimplemented method for " << kind_; + DLOG(FATAL) << "Shouldn't call the unimplemented method for " << kind_; return nullptr; } @@ -445,7 +445,7 @@ class VariableDependencyNode : public PlanNode { std::unique_ptr explain() const override; PlanNode* clone() const override { - LOG(DFATAL) << "Shouldn't call the unimplemented method"; + DLOG(FATAL) << "Shouldn't call the unimplemented method"; return nullptr; } diff --git a/src/graph/util/ParserUtil.cpp b/src/graph/util/ParserUtil.cpp index 1ab8c3b07ce..83a7490aba1 100644 --- a/src/graph/util/ParserUtil.cpp +++ b/src/graph/util/ParserUtil.cpp @@ -62,7 +62,7 @@ void ParserUtil::rewriteLC(QueryContext *qctx, return static_cast(mpp); } break; default: - LOG(DFATAL) << "Unexpected expression kind: " << expr->kind(); + DLOG(FATAL) << "Unexpected expression kind: " << expr->kind(); } return ret; }; diff --git a/src/graph/util/ToJson.cpp b/src/graph/util/ToJson.cpp index 5cc8cbea458..5c6b574690a 100644 --- a/src/graph/util/ToJson.cpp +++ b/src/graph/util/ToJson.cpp @@ -82,7 +82,7 @@ folly::dynamic toJson(const Value &value) { // TODO store to object or array return value.toString(); } - LOG(DFATAL) << "Impossible reach."; + DLOG(FATAL) << "Impossible reach."; return folly::dynamic::object(); } diff --git a/src/graph/validator/LookupValidator.cpp b/src/graph/validator/LookupValidator.cpp index 40694e1348b..4f185327280 100644 --- a/src/graph/validator/LookupValidator.cpp +++ b/src/graph/validator/LookupValidator.cpp @@ -534,7 +534,7 @@ Expression* LookupValidator::reverseRelKind(RelationalExpression* expr) { reversedKind = ExprKind::kRelLE; break; default: - LOG(DFATAL) << "Invalid relational expression kind: " << static_cast(kind); + DLOG(FATAL) << "Invalid relational expression kind: " << static_cast(kind); return expr; } diff --git a/src/graph/validator/Validator.cpp b/src/graph/validator/Validator.cpp index 4ed21d4d436..18b4732273a 100644 --- a/src/graph/validator/Validator.cpp +++ b/src/graph/validator/Validator.cpp @@ -260,10 +260,10 @@ std::unique_ptr Validator::makeValidator(Sentence* sentence, QueryCon case Sentence::Kind::kUnknown: case Sentence::Kind::kReturn: { // nothing - LOG(DFATAL) << "Unimplemented sentence " << kind; + DLOG(FATAL) << "Unimplemented sentence " << kind; } } - LOG(DFATAL) << "Unknown sentence " << static_cast(kind); + DLOG(FATAL) << "Unknown sentence " << static_cast(kind); return std::make_unique(sentence, context); } diff --git a/src/graph/visitor/DeduceTypeVisitor.cpp b/src/graph/visitor/DeduceTypeVisitor.cpp index d8689337d51..f6c3c5dba86 100644 --- a/src/graph/visitor/DeduceTypeVisitor.cpp +++ b/src/graph/visitor/DeduceTypeVisitor.cpp @@ -219,7 +219,7 @@ void DeduceTypeVisitor::visit(UnaryExpression *expr) { break; } default: { - LOG(DFATAL) << "Invalid unary expression kind: " << static_cast(expr->kind()); + DLOG(FATAL) << "Invalid unary expression kind: " << static_cast(expr->kind()); std::stringstream ss; ss << "`" << expr->toString() << "' is invalid unary expression, kind is " << static_cast(expr->kind()) << "."; @@ -284,7 +284,7 @@ void DeduceTypeVisitor::visit(ArithmeticExpression *expr) { break; } default: { - LOG(DFATAL) << "Invalid arithmetic expression kind: " << static_cast(expr->kind()); + DLOG(FATAL) << "Invalid arithmetic expression kind: " << static_cast(expr->kind()); std::stringstream ss; ss << "`" << expr->toString() << "' is invalid arithmetic expression, kind is " << static_cast(expr->kind()) << "."; @@ -430,7 +430,7 @@ void DeduceTypeVisitor::visit(LogicalExpression *expr) { break; } default: { - LOG(DFATAL) << "Invalid logical expression kind: " << static_cast(expr->kind()); + DLOG(FATAL) << "Invalid logical expression kind: " << static_cast(expr->kind()); std::stringstream ss; ss << "`" << expr->toString() << "' is invalid logical expression, kind is " << static_cast(expr->kind()) << "."; diff --git a/src/graph/visitor/ExtractPropExprVisitor.cpp b/src/graph/visitor/ExtractPropExprVisitor.cpp index 44531ddd02f..61e79c11c61 100644 --- a/src/graph/visitor/ExtractPropExprVisitor.cpp +++ b/src/graph/visitor/ExtractPropExprVisitor.cpp @@ -93,7 +93,7 @@ void ExtractPropExprVisitor::visit(UnaryExpression* expr) { break; } default: { - LOG(DFATAL) << "Invalid Kind " << expr->kind(); + DLOG(FATAL) << "Invalid Kind " << expr->kind(); reportError(expr); } } @@ -111,7 +111,7 @@ void ExtractPropExprVisitor::visitPropertyExpr(PropertyExpression* expr) { break; } default: { - LOG(DFATAL) << "Invalid Kind " << expr->kind(); + DLOG(FATAL) << "Invalid Kind " << expr->kind(); reportError(expr); return; } @@ -165,7 +165,7 @@ void ExtractPropExprVisitor::visitVertexEdgePropExpr(PropertyExpression* expr) { break; } default: { - LOG(DFATAL) << "Invalid Kind " << expr->kind(); + DLOG(FATAL) << "Invalid Kind " << expr->kind(); reportError(expr); return; } diff --git a/src/meta/processors/schema/SchemaUtil.cpp b/src/meta/processors/schema/SchemaUtil.cpp index c482b1a5fd8..526efc1fe34 100644 --- a/src/meta/processors/schema/SchemaUtil.cpp +++ b/src/meta/processors/schema/SchemaUtil.cpp @@ -187,7 +187,7 @@ bool SchemaUtil::checkType(std::vector& columns) { return false; // no default so compiler will warning when lack } // switch - LOG(DFATAL) << "Unknown property type " << static_cast(column.get_type().get_type()); + DLOG(FATAL) << "Unknown property type " << static_cast(column.get_type().get_type()); return false; } diff --git a/src/parser/AdminSentences.cpp b/src/parser/AdminSentences.cpp index 99ad3c3a306..f8581e98739 100644 --- a/src/parser/AdminSentences.cpp +++ b/src/parser/AdminSentences.cpp @@ -76,7 +76,7 @@ std::string SpaceOptItem::toString() const { case OptionType::GROUP_NAME: return ""; } - LOG(DFATAL) << "Space parameter illegal"; + DLOG(FATAL) << "Space parameter illegal"; return "Unknown"; } @@ -215,8 +215,8 @@ std::string AddListenerSentence::toString() const { buf += "ELASTICSEARCH "; break; case meta::cpp2::ListenerType::UNKNOWN: - LOG(DFATAL) << "Unknown listener type."; - return "Unknown"; + DLOG(FATAL) << "Unknown listener type."; + return ""; } buf += listeners_->toString(); return buf; @@ -231,8 +231,8 @@ std::string RemoveListenerSentence::toString() const { buf += "ELASTICSEARCH "; break; case meta::cpp2::ListenerType::UNKNOWN: - LOG(DFATAL) << "Unknown listener type."; - return "Unknown"; + DLOG(FATAL) << "Unknown listener type."; + return ""; } return buf; } @@ -309,8 +309,8 @@ std::string AdminJobSentence::toString() const { return str; } } - LOG(DFATAL) << "Unknown job operation " << static_cast(op_); - return "Unknown"; + DLOG(FATAL) << "Unknown job operation " << static_cast(op_); + return ""; } meta::cpp2::JobOp AdminJobSentence::getOp() const { @@ -351,8 +351,8 @@ std::string ShowServiceClientsSentence::toString() const { case meta::cpp2::ExternalServiceType::ELASTICSEARCH: return "SHOW TEXT SEARCH CLIENTS"; default: - LOG(DFATAL) << "Unknown service type " << static_cast(type_); - return "Unknown"; + DLOG(FATAL) << "Unknown service type " << static_cast(type_); + return ""; } } @@ -364,8 +364,8 @@ std::string SignInServiceSentence::toString() const { buf += "SIGN IN TEXT SERVICE "; break; default: - LOG(DFATAL) << "Unknown service type " << static_cast(type_); - return "Unknown"; + DLOG(FATAL) << "Unknown service type " << static_cast(type_); + return ""; } for (auto &client : clients_->clients()) { @@ -404,8 +404,8 @@ std::string SignOutServiceSentence::toString() const { case meta::cpp2::ExternalServiceType::ELASTICSEARCH: return "SIGN OUT TEXT SERVICE"; default: - LOG(DFATAL) << "Unknown service type " << static_cast(type_); - return "Unknown"; + DLOG(FATAL) << "Unknown service type " << static_cast(type_); + return ""; } } diff --git a/src/parser/MaintainSentences.cpp b/src/parser/MaintainSentences.cpp index ea881b09e70..7167a536f49 100644 --- a/src/parser/MaintainSentences.cpp +++ b/src/parser/MaintainSentences.cpp @@ -25,8 +25,8 @@ std::string SchemaPropItem::toString() const { case COMMENT: return folly::stringPrintf("comment = \"%s\"", std::get(propValue_).c_str()); } - LOG(DFATAL) << "Schema property type illegal"; - return "Unknown"; + DLOG(FATAL) << "Schema property type illegal"; + return ""; } std::string SchemaPropList::toString() const { @@ -239,8 +239,8 @@ std::string IndexParamItem::toString() const { case S2_MAX_CELLS: return folly::stringPrintf("s2_max_cells = \"%ld\"", paramValue_.getInt()); } - LOG(DFATAL) << "Index param type illegal"; - return "Unknown"; + DLOG(FATAL) << "Index param type illegal"; + return ""; } std::string IndexParamList::toString() const { diff --git a/src/parser/TraverseSentences.cpp b/src/parser/TraverseSentences.cpp index a773e5e9d82..fcd12a862ea 100644 --- a/src/parser/TraverseSentences.cpp +++ b/src/parser/TraverseSentences.cpp @@ -124,8 +124,8 @@ std::string OrderFactor::toString() const { case DESCEND: return folly::stringPrintf("%s DESC,", expr_->toString().c_str()); default: - LOG(DFATAL) << "Unknown Order Type: " << orderType_; - return "Unknown"; + DLOG(FATAL) << "Unknown Order Type: " << orderType_; + return ""; } } diff --git a/src/storage/GraphStorageServiceHandler.cpp b/src/storage/GraphStorageServiceHandler.cpp index d03ce0bc5dc..d9e6688f0c0 100644 --- a/src/storage/GraphStorageServiceHandler.cpp +++ b/src/storage/GraphStorageServiceHandler.cpp @@ -160,7 +160,7 @@ folly::Future GraphStorageServiceHandler::future_scanEdge( folly::Future GraphStorageServiceHandler::future_getUUID( const cpp2::GetUUIDReq&) { - LOG(DFATAL) << "Unsupported in version 2.0"; + DLOG(FATAL) << "Unsupported in version 2.0"; cpp2::GetUUIDResp resp; cpp2::ResponseCommon result; diff --git a/src/storage/StorageServer.cpp b/src/storage/StorageServer.cpp index 40226d42d95..6bb98f9f76f 100644 --- a/src/storage/StorageServer.cpp +++ b/src/storage/StorageServer.cpp @@ -107,10 +107,10 @@ std::unique_ptr StorageServer::getStoreInstance() { } return nbStore; } else if (FLAGS_store_type == "hbase") { - LOG(DFATAL) << "HBase store has not been implemented"; + DLOG(FATAL) << "HBase store has not been implemented"; return nullptr; } else { - LOG(DFATAL) << "Unknown store type \"" << FLAGS_store_type << "\""; + DLOG(FATAL) << "Unknown store type \"" << FLAGS_store_type << "\""; return nullptr; } return nullptr; @@ -163,7 +163,7 @@ int32_t StorageServer::getAdminStoreSeqId() { newVal.append(reinterpret_cast(&curSeqId), sizeof(int32_t)); auto ret = env_->adminStore_->put(key, newVal); if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(DFATAL) << "Write put in admin-storage seq id " << curSeqId << " failed."; + DLOG(FATAL) << "Write put in admin-storage seq id " << curSeqId << " failed."; return -1; } return curSeqId; diff --git a/src/storage/context/StorageExpressionContext.h b/src/storage/context/StorageExpressionContext.h index a51db60105c..ca84ace9fce 100644 --- a/src/storage/context/StorageExpressionContext.h +++ b/src/storage/context/StorageExpressionContext.h @@ -89,7 +89,7 @@ class StorageExpressionContext final : public ExpressionContext { // Get index of property in variable tuple StatusOr getVarPropIndex(const std::string&, const std::string&) const override { - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Status::Error("Unimplemented"); } @@ -114,7 +114,7 @@ class StorageExpressionContext final : public ExpressionContext { // Get index of property in input tuple StatusOr getInputPropIndex(const std::string&) const override { - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Status::Error("Unimplemented"); } @@ -192,7 +192,7 @@ class StorageExpressionContext final : public ExpressionContext { */ Value getVertex(const std::string& name = "") const override { UNUSED(name); - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Value::kNullBadData; } @@ -202,7 +202,7 @@ class StorageExpressionContext final : public ExpressionContext { * @return Value */ Value getEdge() const override { - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Value::kNullBadData; } diff --git a/src/storage/exec/IndexExprContext.h b/src/storage/exec/IndexExprContext.h index a96b231c261..6dbe7539f9f 100644 --- a/src/storage/exec/IndexExprContext.h +++ b/src/storage/exec/IndexExprContext.h @@ -63,7 +63,7 @@ class IndexExprContext : public ExpressionContext { StatusOr getVarPropIndex(const std::string &var, const std::string &prop) const override { UNUSED(var), UNUSED(prop); - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Status::Error("Unimplemented"); } Value getSrcProp(const std::string &tag, const std::string &prop) const override { @@ -80,7 +80,7 @@ class IndexExprContext : public ExpressionContext { } StatusOr getInputPropIndex(const std::string &prop) const override { UNUSED(prop); - LOG(DFATAL) << "Unimplemented"; + DLOG(FATAL) << "Unimplemented"; return Status::Error("Unimplemented"); } Value getVertex(const std::string &) const override { diff --git a/src/storage/exec/MultiTagNode.h b/src/storage/exec/MultiTagNode.h index 50698469994..9398a556f7b 100644 --- a/src/storage/exec/MultiTagNode.h +++ b/src/storage/exec/MultiTagNode.h @@ -95,17 +95,17 @@ class MultiTagNode : public IterateNode { } folly::StringPiece key() const override { - LOG(DFATAL) << "not allowed to do this"; + DLOG(FATAL) << "not allowed to do this"; return ""; } folly::StringPiece val() const override { - LOG(DFATAL) << "not allowed to do this"; + DLOG(FATAL) << "not allowed to do this"; return ""; } RowReader* reader() const override { - LOG(DFATAL) << "not allowed to do this"; + DLOG(FATAL) << "not allowed to do this"; return nullptr; } diff --git a/src/storage/exec/QueryUtils.h b/src/storage/exec/QueryUtils.h index 42d433c7ae3..cbc685bb1e9 100644 --- a/src/storage/exec/QueryUtils.h +++ b/src/storage/exec/QueryUtils.h @@ -163,7 +163,7 @@ class QueryUtils final { } } default: - LOG(DFATAL) << "Should not read here"; + LOG(FATAL) << "Should not read here"; } return Status::Error(folly::stringPrintf("Invalid property %s", prop.name_.c_str())); } @@ -191,7 +191,7 @@ class QueryUtils final { return tag; } default: - LOG(DFATAL) << "Should not read here"; + LOG(FATAL) << "Should not read here"; } return Status::Error(folly::stringPrintf("Invalid property %s", prop.name_.c_str())); } diff --git a/src/storage/exec/StorageIterator.h b/src/storage/exec/StorageIterator.h index e26833b27a4..195bd8f5c52 100644 --- a/src/storage/exec/StorageIterator.h +++ b/src/storage/exec/StorageIterator.h @@ -173,7 +173,7 @@ class SingleEdgeKeyIterator : public SingleEdgeIterator { } RowReader* reader() const override { - LOG(DFATAL) << "This iterator should not read value"; + DLOG(FATAL) << "This iterator should not read value"; return nullptr; } }; diff --git a/src/storage/transaction/ChainProcessorFactory.cpp b/src/storage/transaction/ChainProcessorFactory.cpp index ed660d60992..ee347f896e2 100644 --- a/src/storage/transaction/ChainProcessorFactory.cpp +++ b/src/storage/transaction/ChainProcessorFactory.cpp @@ -61,7 +61,7 @@ ChainBaseProcessor* ChainProcessorFactory::makeProcessor(StorageEnv* env, break; } case ResumeType::UNKNOWN: { - LOG(DFATAL) << "ResumeType::UNKNOWN: not supposed run here"; + DLOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; return nullptr; } } @@ -80,7 +80,7 @@ ChainBaseProcessor* ChainProcessorFactory::makeProcessor(StorageEnv* env, break; } case ResumeType::UNKNOWN: { - LOG(DFATAL) << "ResumeType::UNKNOWN: not supposed run here"; + DLOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; return nullptr; } } @@ -99,14 +99,14 @@ ChainBaseProcessor* ChainProcessorFactory::makeProcessor(StorageEnv* env, break; } case ResumeType::UNKNOWN: { - LOG(DFATAL) << "ResumeType::UNKNOWN: not supposed run here"; + DLOG(FATAL) << "ResumeType::UNKNOWN: not supposed run here"; return nullptr; } } break; } case RequestType::UNKNOWN: { - LOG(DFATAL) << "RequestType::UNKNOWN: not supposed run here"; + DLOG(FATAL) << "RequestType::UNKNOWN: not supposed run here"; return nullptr; } } diff --git a/src/storage/transaction/ConsistUtil.cpp b/src/storage/transaction/ConsistUtil.cpp index d1aa3c0a3cb..3ba7ba9bdd5 100644 --- a/src/storage/transaction/ConsistUtil.cpp +++ b/src/storage/transaction/ConsistUtil.cpp @@ -72,7 +72,7 @@ RequestType ConsistUtil::parseType(folly::StringPiece val) { case 'd': return RequestType::DELETE; default: - LOG(DFATAL) << "should not happen, identifier is " << identifier; + LOG(FATAL) << "should not happen, identifier is " << identifier; return RequestType::UNKNOWN; } } From 1614c956df2b6fdc8064a83bf3260230a69f75d9 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 5 Jan 2023 12:12:00 +0800 Subject: [PATCH 08/51] Meta upgrade (#5174) * Meta upgrade remove all fulltext index when upgrade from V3 to V3_4 because of refacting of fulltext index * fix bug Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/clients/meta/MetaClient.h | 2 +- src/daemons/MetaDaemonInit.cpp | 12 ++-- src/meta/MetaVersionMan.cpp | 104 ++++++--------------------- src/meta/MetaVersionMan.h | 5 +- src/tools/meta-dump/MetaDumpTool.cpp | 3 +- 5 files changed, 34 insertions(+), 92 deletions(-) diff --git a/src/clients/meta/MetaClient.h b/src/clients/meta/MetaClient.h index 87bf6e9a2e7..c00205f7dfc 100644 --- a/src/clients/meta/MetaClient.h +++ b/src/clients/meta/MetaClient.h @@ -774,7 +774,7 @@ class MetaClient : public BaseMetaClient { std::atomic metadLastUpdateTime_{0}; int64_t metaServerVersion_{-1}; - static constexpr int64_t EXPECT_META_VERSION = 3; + static constexpr int64_t EXPECT_META_VERSION = 4; // leadersLock_ is used to protect leadersInfo folly::SharedMutex leadersLock_; diff --git a/src/daemons/MetaDaemonInit.cpp b/src/daemons/MetaDaemonInit.cpp index 69f17b44973..60069095824 100644 --- a/src/daemons/MetaDaemonInit.cpp +++ b/src/daemons/MetaDaemonInit.cpp @@ -147,16 +147,18 @@ std::unique_ptr initKV(std::vector p LOG(ERROR) << "Meta version is invalid"; return nullptr; } else if (version == nebula::meta::MetaVersion::V1) { - LOG(ERROR) << "Can't upgrade meta from V1 to V3"; + LOG(ERROR) << "Can't upgrade meta from V1 to V3_4"; return nullptr; } else if (version == nebula::meta::MetaVersion::V2) { - auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(engine); + LOG(ERROR) << "Can't upgrade meta from V2 to V3_4"; + return nullptr; + } else if (version == nebula::meta::MetaVersion::V3) { + auto ret = nebula::meta::MetaVersionMan::updateMetaV3ToV3_4(engine); if (!ret.ok()) { - LOG(ERROR) << "Update meta from V2 to V3 failed " << ret; + LOG(ERROR) << "Update meta from V3 to V3_4 failed " << ret; return nullptr; } - - nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V3); + nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V3_4); } LOG(INFO) << "Nebula store init succeeded, clusterId " << gClusterId; diff --git a/src/meta/MetaVersionMan.cpp b/src/meta/MetaVersionMan.cpp index 41acf1f001e..78bae55d102 100644 --- a/src/meta/MetaVersionMan.cpp +++ b/src/meta/MetaVersionMan.cpp @@ -43,10 +43,10 @@ MetaVersion MetaVersionMan::getVersionByHost(kvstore::KVStore* kv) { } if (iter->valid()) { auto v1KeySize = hostPrefix.size() + sizeof(int64_t); - return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3; + return (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3_4; } // No hosts exists, regard as version 3 - return MetaVersion::V3; + return MetaVersion::V3_4; } // static @@ -58,7 +58,7 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion v return code == nebula::cpp2::ErrorCode::SUCCEEDED; } -Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) { +Status MetaVersionMan::updateMetaV3ToV3_4(kvstore::KVEngine* engine) { CHECK_NOTNULL(engine); auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()); @@ -75,7 +75,7 @@ Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) { return Status::Error("Create snapshot failed"); } - auto status = doUpgradeV2ToV3(engine); + auto status = doUpgradeV3ToV3_4(engine); if (!status.ok()) { // rollback by snapshot return status; @@ -89,87 +89,25 @@ Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) { return Status::OK(); } -Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVEngine* engine) { - MetaDataUpgrade upgrader(engine); - // Step 1: Upgrade HeartBeat into machine list - { - // collect all hosts association with zone - std::vector zoneHosts; - const auto& zonePrefix = MetaKeyUtils::zonePrefix(); - std::unique_ptr zoneIter; - auto code = engine->prefix(zonePrefix, &zoneIter); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Get active hosts failed"; - return Status::Error("Get hosts failed"); - } - - while (zoneIter->valid()) { - auto hosts = MetaKeyUtils::parseZoneHosts(zoneIter->val()); - if (!hosts.empty()) { - zoneHosts.insert(zoneHosts.end(), hosts.begin(), hosts.end()); - } - zoneIter->next(); - } - - const auto& prefix = MetaKeyUtils::hostPrefix(); - std::unique_ptr iter; - code = engine->prefix(prefix, &iter); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Get active hosts failed"; - return Status::Error("Get hosts failed"); - } - - std::vector data; - while (iter->valid()) { - auto info = HostInfo::decode(iter->val()); - - if (info.role_ == meta::cpp2::HostRole::STORAGE) { - // Save the machine information - auto host = MetaKeyUtils::parseHostKey(iter->key()); - auto machineKey = MetaKeyUtils::machineKey(host.host, host.port); - data.emplace_back(std::move(machineKey), ""); - - auto hostIt = std::find(zoneHosts.begin(), zoneHosts.end(), host); - if (hostIt == zoneHosts.end()) { - // Save the zone information - auto zoneName = folly::stringPrintf("default_zone_%s_%d", host.host.c_str(), host.port); - auto zoneKey = MetaKeyUtils::zoneKey(std::move(zoneName)); - auto zoneVal = MetaKeyUtils::zoneVal({host}); - data.emplace_back(std::move(zoneKey), std::move(zoneVal)); - } - } - iter->next(); - } - auto status = upgrader.saveMachineAndZone(std::move(data)); - if (!status.ok()) { - LOG(INFO) << status; - return status; - } +Status MetaVersionMan::doUpgradeV3ToV3_4(kvstore::KVEngine* engine) { + std::unique_ptr fulltextIter; + auto code = engine->prefix(MetaKeyUtils::fulltextIndexPrefix(), &fulltextIter); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Upgrade meta failed"; + return Status::Error("Update meta failed"); } - - // Step 2: Update Create space properties about Group - { - const auto& prefix = MetaKeyUtils::spacePrefix(); - std::unique_ptr iter; - auto code = engine->prefix(prefix, &iter); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << "Get spaces failed"; - return Status::Error("Get spaces failed"); - } - - while (iter->valid()) { - if (FLAGS_print_info) { - upgrader.printSpacesV2(iter->val()); - } - auto status = upgrader.rewriteSpacesV2ToV3(iter->key(), iter->val()); - if (!status.ok()) { - LOG(INFO) << status; - return status; - } - iter->next(); - } + std::vector fulltextList; + while (fulltextIter->valid()) { + fulltextList.push_back(fulltextIter->key().toString()); + fulltextIter->next(); } - if (!setMetaVersionToKV(engine, MetaVersion::V3)) { + code = engine->multiRemove(fulltextList); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(ERROR) << "Upgrade meta failed"; + return Status::Error("Upgrade meta failed"); + } + + if (!setMetaVersionToKV(engine, MetaVersion::V3_4)) { return Status::Error("Persist meta version failed"); } else { return Status::OK(); diff --git a/src/meta/MetaVersionMan.h b/src/meta/MetaVersionMan.h index 99a9c297df9..be0c0d727db 100644 --- a/src/meta/MetaVersionMan.h +++ b/src/meta/MetaVersionMan.h @@ -19,6 +19,7 @@ enum class MetaVersion { V1 = 1, V2 = 2, V3 = 3, + V3_4 = 4, }; /** @@ -38,12 +39,12 @@ class MetaVersionMan final { static bool setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion version); - static Status updateMetaV2ToV3(kvstore::KVEngine* engine); + static Status updateMetaV3ToV3_4(kvstore::KVEngine* engine); private: static MetaVersion getVersionByHost(kvstore::KVStore* kv); - static Status doUpgradeV2ToV3(kvstore::KVEngine* engine); + static Status doUpgradeV3ToV3_4(kvstore::KVEngine* engine); }; } // namespace meta diff --git a/src/tools/meta-dump/MetaDumpTool.cpp b/src/tools/meta-dump/MetaDumpTool.cpp index 23a9f8e814e..52074aa3756 100644 --- a/src/tools/meta-dump/MetaDumpTool.cpp +++ b/src/tools/meta-dump/MetaDumpTool.cpp @@ -42,6 +42,7 @@ class MetaDumper { V1 = 1, V2 = 2, V3 = 3, + V3_4 = 4, }; prefix = "__meta_version__"; @@ -60,7 +61,7 @@ class MetaDumper { while (iter->Valid() && iter->key().starts_with(prefix)) { found = true; auto v1KeySize = prefix.size() + sizeof(int64_t); - auto version = (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3; + auto version = (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3_4; LOG(INFO) << "Meta version=" << static_cast(version); iter->Next(); break; From 635f3764fb81b0782c841927d6556f849a6c14ef Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Thu, 5 Jan 2023 15:54:58 +0800 Subject: [PATCH 09/51] Fix pattern expression with same edge variable (#5192) * Fix pattern expression with same edge variable add tck fmt * add tck --- src/common/function/FunctionManager.cpp | 9 +++ src/graph/planner/match/SegmentsConnector.cpp | 4 +- src/graph/validator/MatchValidator.cpp | 19 ++++-- .../features/bugfix/AliasTypeDeduce.feature | 4 +- tests/tck/features/match/PathExpr.feature | 28 ++++++-- .../match/PathExprRefLocalVariable.feature | 64 +++++++++++++++++++ 6 files changed, 116 insertions(+), 12 deletions(-) diff --git a/src/common/function/FunctionManager.cpp b/src/common/function/FunctionManager.cpp index 74a762a93d9..11d41741cc8 100644 --- a/src/common/function/FunctionManager.cpp +++ b/src/common/function/FunctionManager.cpp @@ -1867,6 +1867,15 @@ FunctionManager::FunctionManager() { case Value::Type::EDGE: { return value.getEdge().id(); } + // The root cause is the edge-type data format of Traverse executor + case Value::Type::LIST: { + auto &edges = value.getList().values; + if (edges.size() == 1 && edges[0].isEdge()) { + return edges[0].getEdge().id(); + } else { + return args[0]; + } + } default: { // Join on the origin type return args[0]; diff --git a/src/graph/planner/match/SegmentsConnector.cpp b/src/graph/planner/match/SegmentsConnector.cpp index cc29fb85cb1..813a4208eaf 100644 --- a/src/graph/planner/match/SegmentsConnector.cpp +++ b/src/graph/planner/match/SegmentsConnector.cpp @@ -79,7 +79,7 @@ SubPlan SegmentsConnector::rollUpApply(CypherClauseContextBase* ctx, std::vector compareProps; for (const auto& col : path.compareVariables) { compareProps.emplace_back(FunctionCallExpression::make( - qctx->objPool(), "id", {InputPropertyExpression::make(qctx->objPool(), col)})); + qctx->objPool(), "_joinkey", {InputPropertyExpression::make(qctx->objPool(), col)})); } InputPropertyExpression* collectProp = InputPropertyExpression::make(qctx->objPool(), collectCol); auto* rollUpApply = RollUpApply::make( @@ -104,7 +104,7 @@ SubPlan SegmentsConnector::rollUpApply(CypherClauseContextBase* ctx, std::vector keyProps; for (const auto& col : path.compareVariables) { keyProps.emplace_back(FunctionCallExpression::make( - qctx->objPool(), "id", {InputPropertyExpression::make(qctx->objPool(), col)})); + qctx->objPool(), "_joinkey", {InputPropertyExpression::make(qctx->objPool(), col)})); } auto* patternApply = PatternApply::make( qctx, left.root, DCHECK_NOTNULL(right.root), std::move(keyProps), path.isAntiPred); diff --git a/src/graph/validator/MatchValidator.cpp b/src/graph/validator/MatchValidator.cpp index 56c2505deba..e47de251cf7 100644 --- a/src/graph/validator/MatchValidator.cpp +++ b/src/graph/validator/MatchValidator.cpp @@ -1240,7 +1240,7 @@ Status MatchValidator::validatePathInWhere( matchPath.alias()->c_str()); } if (find->second != AliasType::kPath) { - return Status::SemanticError("Alias `%s' should be Path, but got type '%s", + return Status::SemanticError("`%s' is defined with type %s, but referenced with type Path", matchPath.alias()->c_str(), AliasTypeName::get(find->second).c_str()); } @@ -1258,7 +1258,7 @@ Status MatchValidator::validatePathInWhere( node->alias().c_str()); } if (find->second != AliasType::kNode) { - return Status::SemanticError("Alias `%s' should be Node, but got type '%s", + return Status::SemanticError("`%s' is defined with type %s, but referenced with type Node", node->alias().c_str(), AliasTypeName::get(find->second).c_str()); } @@ -1272,11 +1272,17 @@ Status MatchValidator::validatePathInWhere( "PatternExpression are not allowed to introduce new variables: `%s'.", edge->alias().c_str()); } - if (find->second != AliasType::kEdge) { - return Status::SemanticError("Alias `%s' should be Edge, but got type '%s'", + if (!edge->range() && find->second != AliasType::kEdge) { + return Status::SemanticError("`%s' is defined with type %s, but referenced with type Edge", edge->alias().c_str(), AliasTypeName::get(find->second).c_str()); } + if (edge->range() && find->second != AliasType::kEdgeList) { + return Status::SemanticError( + "`%s' is defined with type %s, but referenced with type EdgeList", + edge->alias().c_str(), + AliasTypeName::get(find->second).c_str()); + } } } return Status::OK(); @@ -1290,6 +1296,11 @@ Status MatchValidator::validatePathInWhere( pathInfo.compareVariables.emplace_back(node->alias()); } } + for (const auto &edge : path->edges()) { + if (edge->alias()[0] != '_') { + pathInfo.compareVariables.emplace_back(edge->alias()); + } + } pathInfo.collectVariable = *path->alias(); pathInfo.rollUpApply = true; return Status::OK(); diff --git a/tests/tck/features/bugfix/AliasTypeDeduce.feature b/tests/tck/features/bugfix/AliasTypeDeduce.feature index 958c2183788..4c9caaa6b1d 100644 --- a/tests/tck/features/bugfix/AliasTypeDeduce.feature +++ b/tests/tck/features/bugfix/AliasTypeDeduce.feature @@ -34,7 +34,7 @@ Feature: Test extract filter """ Then the result should be, in any order: | count(c) | - | 3225 | + | 49 | When executing query: """ match p=(a:player)-[e:like*1..3]->(b) @@ -46,4 +46,4 @@ Feature: Test extract filter """ Then the result should be, in any order: | count(c) | - | 3225 | + | 49 | diff --git a/tests/tck/features/match/PathExpr.feature b/tests/tck/features/match/PathExpr.feature index d4366d179c6..d115285b67d 100644 --- a/tests/tck/features/match/PathExpr.feature +++ b/tests/tck/features/match/PathExpr.feature @@ -50,22 +50,42 @@ Feature: Basic match """ MATCH (v:player) WITH (v)-[v]-() AS p RETURN p """ - Then a SemanticError should be raised at runtime: Alias `v' should be Edge, but got type 'Node' + Then a SemanticError should be raised at runtime: `v' is defined with type Node, but referenced with type Edge When executing query: """ MATCH (v:player) UNWIND (v)-[v]-() AS p RETURN p """ - Then a SemanticError should be raised at runtime: Alias `v' should be Edge, but got type 'Node' + Then a SemanticError should be raised at runtime: `v' is defined with type Node, but referenced with type Edge When executing query: """ MATCH (v:player) WHERE (v)-[v]-() RETURN v """ - Then a SemanticError should be raised at runtime: Alias `v' should be Edge, but got type 'Node' + Then a SemanticError should be raised at runtime: `v' is defined with type Node, but referenced with type Edge When executing query: """ MATCH (v:player) RETURN (v)-[v]-() """ - Then a SemanticError should be raised at runtime: Alias `v' should be Edge, but got type 'Node' + Then a SemanticError should be raised at runtime: `v' is defined with type Node, but referenced with type Edge + When executing query: + """ + MATCH (v:player)-[e*3]->() RETURN (v)-[e]-() + """ + Then a SemanticError should be raised at runtime: `e' is defined with type EdgeList, but referenced with type Edge + When executing query: + """ + MATCH (v:player)-[e]->() RETURN (v)-[e*1..3]-() + """ + Then a SemanticError should be raised at runtime: `e' is defined with type Edge, but referenced with type EdgeList + When executing query: + """ + MATCH (v:player)-[e]->() RETURN (e)-[*1..3]-() + """ + Then a SemanticError should be raised at runtime: `e' is defined with type Edge, but referenced with type Node + When executing query: + """ + MATCH (v:player)-[e*3]->() RETURN (e)-[*1..3]-() + """ + Then a SemanticError should be raised at runtime: `e' is defined with type EdgeList, but referenced with type Node Scenario: In Where When executing query: diff --git a/tests/tck/features/match/PathExprRefLocalVariable.feature b/tests/tck/features/match/PathExprRefLocalVariable.feature index 3c3e4b07cbe..02355c3d134 100644 --- a/tests/tck/features/match/PathExprRefLocalVariable.feature +++ b/tests/tck/features/match/PathExprRefLocalVariable.feature @@ -100,6 +100,43 @@ Feature: Path expression reference local defined variables | name | | "Tim Duncan" | | "Tim Duncan" | + When executing query: + """ + MATCH (v:player)-[e:like]->(n) WHERE (n)-[e]->(:player) RETURN v + """ + Then the result should be, in any order: + | v | + When executing query: + """ + MATCH (v:player)-[e]->(n) WHERE ()-[e]->(:player) and e.likeness<80 RETURN distinct v.player.name AS vname + """ + Then the result should be, in any order: + | vname | + | "Kyrie Irving" | + | "LaMarcus Aldridge" | + | "Dirk Nowitzki" | + | "Rudy Gay" | + | "Danny Green" | + | "Blake Griffin" | + | "Marco Belinelli" | + | "Vince Carter" | + | "Rajon Rondo" | + | "Ray Allen" | + + @skip + Scenario: Fix after issue #2045 + When executing query: + """ + MATCH (v:player)-[e:like*1..3]->(n) WHERE (n)-[e*1..4]->(:player) return v + """ + Then the result should be, in any order: + | v | + When executing query: + """ + MATCH (v:player)-[e:like*3]->(n) WHERE id(v)=="Tim Duncan" and (n)-[e*3]->(:player) return v + """ + Then the result should be, in any order: + | v | Scenario: In With When executing query: @@ -182,6 +219,26 @@ Feature: Path expression reference local defined variables | p | | [[<("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})>]] | | [[<("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})>]] | + When executing query: + """ + MATCH (v:player)-[e]->(n) WITH ()-[e{likeness:80}]->(:player) AS p1, ()-[e]-(:team) AS p2, v.player.name AS vname WHERE size(p1)>0 RETURN distinct * ORDER BY vname + """ + Then the result should be, in any order, with relax comparison: + | p1 | p2 | vname | + | [<("Aron Baynes" :player{age: 32, name: "Aron Baynes"})-[:like@0 {likeness: 80}]->("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"})>] | [] | "Aron Baynes" | + | [<("Ben Simmons" :player{age: 22, name: "Ben Simmons"})-[:like@0 {likeness: 80}]->("Joel Embiid" :player{age: 25, name: "Joel Embiid"})>] | [] | "Ben Simmons" | + | [<("Boris Diaw" :player{age: 36, name: "Boris Diaw"})-[:like@0 {likeness: 80}]->("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"})>] | [] | "Boris Diaw" | + | [<("Boris Diaw" :player{age: 36, name: "Boris Diaw"})-[:like@0 {likeness: 80}]->("Tony Parker" :player{age: 36, name: "Tony Parker"})>] | [] | "Boris Diaw" | + | [<("Damian Lillard" :player{age: 28, name: "Damian Lillard"})-[:like@0 {likeness: 80}]->("LaMarcus Aldridge" :player{age: 33, name: "LaMarcus Aldridge"})>] | [] | "Damian Lillard" | + | [<("Danny Green" :player{age: 31, name: "Danny Green"})-[:like@0 {likeness: 80}]->("LeBron James" :player{age: 34, name: "LeBron James"})>] | [] | "Danny Green" | + | [<("Dirk Nowitzki" :player{age: 40, name: "Dirk Nowitzki"})-[:like@0 {likeness: 80}]->("Steve Nash" :player{age: 45, name: "Steve Nash"})>] | [] | "Dirk Nowitzki" | + | [<("Dirk Nowitzki" :player{age: 40, name: "Dirk Nowitzki"})-[:like@0 {likeness: 80}]->("Jason Kidd" :player{age: 45, name: "Jason Kidd"})>] | [] | "Dirk Nowitzki" | + | [<("James Harden" :player{age: 29, name: "James Harden"})-[:like@0 {likeness: 80}]->("Russell Westbrook" :player{age: 30, name: "Russell Westbrook"})>] | [] | "James Harden" | + | [<("Jason Kidd" :player{age: 45, name: "Jason Kidd"})-[:like@0 {likeness: 80}]->("Vince Carter" :player{age: 42, name: "Vince Carter"})>] | [] | "Jason Kidd" | + | [<("Joel Embiid" :player{age: 25, name: "Joel Embiid"})-[:like@0 {likeness: 80}]->("Ben Simmons" :player{age: 22, name: "Ben Simmons"})>] | [] | "Joel Embiid" | + | [<("Luka Doncic" :player{age: 20, name: "Luka Doncic"})-[:like@0 {likeness: 80}]->("James Harden" :player{age: 29, name: "James Harden"})>] | [] | "Luka Doncic" | + | [<("Shaquille O'Neal" :player{age: 47, name: "Shaquille O'Neal"})-[:like@0 {likeness: 80}]->("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"})>] | [] | "Shaquille O'Neal" | + | [<("Tiago Splitter" :player{age: 34, name: "Tiago Splitter"})-[:like@0 {likeness: 80}]->("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"})>] | [] | "Tiago Splitter" | Scenario: In Unwind When executing query: @@ -192,3 +249,10 @@ Feature: Path expression reference local defined variables | p | | [<("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})>] | | [<("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})>] | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*1..3]->(n), (t:team {name: "Spurs"}) WITH v, e, collect(distinct n) AS ns UNWIND [n in ns | ()-[e*2..4]->(n:player)] AS p RETURN count(p) AS count + """ + Then the result should be, in any order: + | count | + | 11 | From 6d1bc5de831c91a011abf7aa021b2871eb8691a3 Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:29:13 +0800 Subject: [PATCH 10/51] Fix memory leak, remove toss gflag (#5204) * remove toss gflag * fix memory leak * loose wait job finish time Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/graph/executor/mutate/DeleteExecutor.cpp | 2 +- src/graph/executor/mutate/InsertExecutor.cpp | 2 +- src/graph/executor/mutate/UpdateExecutor.cpp | 2 +- src/graph/service/GraphFlags.cpp | 1 - src/graph/service/GraphFlags.h | 1 - src/graph/validator/MutateValidator.cpp | 40 ++++++++----------- .../processors/index/FTIndexProcessor.cpp | 14 +++---- tests/tck/conftest.py | 2 +- 8 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/graph/executor/mutate/DeleteExecutor.cpp b/src/graph/executor/mutate/DeleteExecutor.cpp index 05a12a05ea6..71fee56ec86 100644 --- a/src/graph/executor/mutate/DeleteExecutor.cpp +++ b/src/graph/executor/mutate/DeleteExecutor.cpp @@ -215,7 +215,7 @@ folly::Future DeleteEdgesExecutor::deleteEdges() { auto plan = qctx()->plan(); StorageClient::CommonRequestParam param( spaceId, qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled()); - param.useExperimentalFeature = FLAGS_enable_experimental_feature && FLAGS_enable_toss; + param.useExperimentalFeature = false; return qctx() ->getStorageClient() ->deleteEdges(param, std::move(edgeKeys)) diff --git a/src/graph/executor/mutate/InsertExecutor.cpp b/src/graph/executor/mutate/InsertExecutor.cpp index 53076051aeb..f41e867b39e 100644 --- a/src/graph/executor/mutate/InsertExecutor.cpp +++ b/src/graph/executor/mutate/InsertExecutor.cpp @@ -61,7 +61,7 @@ folly::Future InsertEdgesExecutor::insertEdges() { auto plan = qctx()->plan(); StorageClient::CommonRequestParam param( ieNode->getSpace(), qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled()); - param.useExperimentalFeature = FLAGS_enable_experimental_feature && FLAGS_enable_toss; + param.useExperimentalFeature = false; return qctx() ->getStorageClient() ->addEdges(param, diff --git a/src/graph/executor/mutate/UpdateExecutor.cpp b/src/graph/executor/mutate/UpdateExecutor.cpp index 355fd40e14f..89d6519cefe 100644 --- a/src/graph/executor/mutate/UpdateExecutor.cpp +++ b/src/graph/executor/mutate/UpdateExecutor.cpp @@ -105,7 +105,7 @@ folly::Future UpdateEdgeExecutor::execute() { auto plan = qctx()->plan(); StorageClient::CommonRequestParam param( ueNode->getSpaceId(), qctx()->rctx()->session()->id(), plan->id(), plan->isProfileEnabled()); - param.useExperimentalFeature = FLAGS_enable_experimental_feature && FLAGS_enable_toss; + param.useExperimentalFeature = false; return qctx() ->getStorageClient() ->updateEdge(param, diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp index 18ce8be9471..7ffce40a526 100644 --- a/src/graph/service/GraphFlags.cpp +++ b/src/graph/service/GraphFlags.cpp @@ -70,7 +70,6 @@ DEFINE_bool(disable_octal_escape_char, " in next version to ensure compatibility with cypher."); DEFINE_bool(enable_experimental_feature, false, "Whether to enable experimental feature"); -DEFINE_bool(enable_toss, false, "Whether to enable toss feature"); DEFINE_bool(enable_data_balance, true, "Whether to enable data balance feature"); DEFINE_int32(num_rows_to_check_memory, 1024, "number rows to check memory"); diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h index 6cb54849762..f7a4ecf2416 100644 --- a/src/graph/service/GraphFlags.h +++ b/src/graph/service/GraphFlags.h @@ -54,7 +54,6 @@ DECLARE_bool(optimize_appendvertice); DECLARE_int64(max_allowed_connections); DECLARE_bool(enable_experimental_feature); -DECLARE_bool(enable_toss); DECLARE_bool(enable_data_balance); DECLARE_bool(enable_client_white_list); diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp index a2d3f798618..6c76ac85c43 100644 --- a/src/graph/validator/MutateValidator.cpp +++ b/src/graph/validator/MutateValidator.cpp @@ -211,8 +211,7 @@ Status InsertEdgesValidator::check() { // Check validity of vertices data. // Check edge key type, check properties value, fill to NewEdge structure. Status InsertEdgesValidator::prepareEdges() { - auto size = - FLAGS_enable_experimental_feature && FLAGS_enable_toss ? rows_.size() : rows_.size() * 2; + auto size = rows_.size() * 2; edges_.reserve(size); size_t fieldNum = schema_->getNumFields(); @@ -297,7 +296,7 @@ Status InsertEdgesValidator::prepareEdges() { edge.key_ref() = key; edge.props_ref() = std::move(entirePropValues); edges_.emplace_back(edge); - if (!(FLAGS_enable_experimental_feature && FLAGS_enable_toss)) { + { // inbound key.src_ref() = dstId; key.dst_ref() = srcId; @@ -892,26 +891,21 @@ Status UpdateEdgeValidator::toPlan() { {}, condition_, {}); - if ((FLAGS_enable_experimental_feature && FLAGS_enable_toss)) { - root_ = outNode; - tail_ = root_; - } else { - auto *inNode = UpdateEdge::make(qctx_, - outNode, - spaceId_, - std::move(name_), - std::move(dstId_), - std::move(srcId_), - -edgeType_, - rank_, - insertable_, - std::move(updatedProps_), - std::move(returnProps_), - std::move(condition_), - std::move(yieldColNames_)); - root_ = inNode; - tail_ = outNode; - } + auto *inNode = UpdateEdge::make(qctx_, + outNode, + spaceId_, + std::move(name_), + std::move(dstId_), + std::move(srcId_), + -edgeType_, + rank_, + insertable_, + std::move(updatedProps_), + std::move(returnProps_), + std::move(condition_), + std::move(yieldColNames_)); + root_ = inNode; + tail_ = outNode; return Status::OK(); } diff --git a/src/meta/processors/index/FTIndexProcessor.cpp b/src/meta/processors/index/FTIndexProcessor.cpp index 9771187ecfa..d5df8d97268 100644 --- a/src/meta/processors/index/FTIndexProcessor.cpp +++ b/src/meta/processors/index/FTIndexProcessor.cpp @@ -173,13 +173,6 @@ void DropFTIndexProcessor::process(const cpp2::DropFTIndexReq& req) { return; } - auto batchHolder = std::make_unique(); - batchHolder->remove(std::move(indexKey)); - auto timeInMilliSec = time::WallClock::fastNowInMilliSec(); - LastUpdateTimeMan::update(batchHolder.get(), timeInMilliSec); - auto batch = encodeBatchValue(std::move(batchHolder)->getBatch()); - doBatchOperation(std::move(batch)); - const auto& serviceKey = MetaKeyUtils::serviceKey(cpp2::ExternalServiceType::ELASTICSEARCH); auto getRet = doGet(serviceKey); if (!nebula::ok(getRet)) { @@ -213,6 +206,13 @@ void DropFTIndexProcessor::process(const cpp2::DropFTIndexReq& req) { onFinished(); return; } + + auto batchHolder = std::make_unique(); + batchHolder->remove(std::move(indexKey)); + auto timeInMilliSec = time::WallClock::fastNowInMilliSec(); + LastUpdateTimeMan::update(batchHolder.get(), timeInMilliSec); + auto batch = encodeBatchValue(std::move(batchHolder)->getBatch()); + doBatchOperation(std::move(batch)); } void ListFTIndexesProcessor::process(const cpp2::ListFTIndexesReq&) { diff --git a/tests/tck/conftest.py b/tests/tck/conftest.py index 3895e98285a..03f159ad36f 100644 --- a/tests/tck/conftest.py +++ b/tests/tck/conftest.py @@ -86,7 +86,7 @@ def running_or_queue_row(row): def wait_all_jobs_finished(sess, jobs=[]): - times = 4 * get_running_jobs(sess) + times = 5 * get_running_jobs(sess) while jobs and times > 0: jobs = [job for job in jobs if not is_job_finished(sess, job)] time.sleep(1) From aa8bde0f4d356aa897c96aa31221bc8d3d11161d Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:39:59 +0800 Subject: [PATCH 11/51] Add max_sessions_per_ip_per_user to default config file (#5207) --- conf/nebula-graphd.conf.default | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default index feb9e273a21..6f184327df6 100644 --- a/conf/nebula-graphd.conf.default +++ b/conf/nebula-graphd.conf.default @@ -93,3 +93,7 @@ # if use balance data feature, only work if enable_experimental_feature is true --enable_data_balance=true + +########## session ########## +# Maximum number of sessions that can be created per IP and per user +--max_sessions_per_ip_per_user=300 From f840eea132dcd3805866325ac5b230796fc4bd24 Mon Sep 17 00:00:00 2001 From: liwenhui-soul <38217397+liwenhui-soul@users.noreply.github.com> Date: Thu, 5 Jan 2023 20:29:58 +0800 Subject: [PATCH 12/51] minor bug for adminTaskManager (#5195) --- src/storage/admin/AdminTaskManager.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/storage/admin/AdminTaskManager.cpp b/src/storage/admin/AdminTaskManager.cpp index ba285c96a41..80985f17a40 100644 --- a/src/storage/admin/AdminTaskManager.cpp +++ b/src/storage/admin/AdminTaskManager.cpp @@ -122,7 +122,7 @@ void AdminTaskManager::handleUnreportedTasks() { jId, tId, fut.value().status().toString()); - if (fut.value().status() == Status::Error("Space not existed!")) { + if (fut.value().status() == Status::SpaceNotFound("Space not existed!")) { // space has been dropped, remove the task status. keys.emplace_back(key.data(), key.size()); } else { @@ -135,13 +135,10 @@ void AdminTaskManager::handleUnreportedTasks() { jId, tId, apache::thrift::util::enumNameSafe(rc)); - if (rc == nebula::cpp2::ErrorCode::E_LEADER_CHANGED || - rc == nebula::cpp2::ErrorCode::E_STORE_FAILURE) { - ifAnyUnreported_ = true; - continue; - } else { + if (rc == nebula::cpp2::ErrorCode::SUCCEEDED) { keys.emplace_back(key.data(), key.size()); - break; + } else { + ifAnyUnreported_ = true; } } env_->adminStore_->multiRemove(keys); From 9e71515cb34ff3580424286c19364ef97461ed41 Mon Sep 17 00:00:00 2001 From: Alex Xing <90179377+SuperYoko@users.noreply.github.com> Date: Fri, 6 Jan 2023 09:54:38 +0800 Subject: [PATCH 13/51] modify jobmanager ut (#5175) * modify jobmanager ut * add expired ut * avoid recover expired job * add ut * address review * move status Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/meta/processors/job/JobDescription.cpp | 6 + src/meta/processors/job/JobExecutor.h | 9 +- src/meta/processors/job/JobManager.cpp | 31 +- src/meta/processors/job/JobManager.h | 7 +- src/meta/processors/job/JobStatus.cpp | 4 + src/meta/processors/job/JobStatus.h | 2 + src/meta/test/JobManagerTest.cpp | 1207 +++++++++++++++++++- 7 files changed, 1210 insertions(+), 56 deletions(-) diff --git a/src/meta/processors/job/JobDescription.cpp b/src/meta/processors/job/JobDescription.cpp index 37beaa40aa0..1e64c890faf 100644 --- a/src/meta/processors/job/JobDescription.cpp +++ b/src/meta/processors/job/JobDescription.cpp @@ -11,6 +11,7 @@ #include #include "common/utils/MetaKeyUtils.h" +#include "interface/gen-cpp2/meta_types.h" #include "kvstore/KVIterator.h" #include "meta/processors/Common.h" @@ -83,6 +84,11 @@ cpp2::JobDesc JobDescription::toJobDesc() { } bool JobDescription::setStatus(Status newStatus, bool force) { + if (JobStatus::notSetable(status_)) { + // no need to change time. + return status_ == newStatus; + } + if (JobStatus::laterThan(status_, newStatus) && !force) { return false; } diff --git a/src/meta/processors/job/JobExecutor.h b/src/meta/processors/job/JobExecutor.h index b5cafb234fc..7be816f3598 100644 --- a/src/meta/processors/job/JobExecutor.h +++ b/src/meta/processors/job/JobExecutor.h @@ -19,7 +19,7 @@ namespace meta { class JobExecutor { public: JobExecutor() = default; - explicit JobExecutor(kvstore::KVStore* kv, GraphSpaceID space) : kvstore_(kv), space_(space) {} + JobExecutor(kvstore::KVStore* kv, GraphSpaceID space) : kvstore_(kv), space_(space) {} virtual ~JobExecutor() = default; /** @@ -99,9 +99,10 @@ class JobExecutor { class JobExecutorFactory { public: - static std::unique_ptr createJobExecutor(const JobDescription& jd, - kvstore::KVStore* store, - AdminClient* client); + virtual ~JobExecutorFactory() = default; + virtual std::unique_ptr createJobExecutor(const JobDescription& jd, + kvstore::KVStore* store, + AdminClient* client); }; } // namespace meta diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp index a2119e0d39d..c9d31c417fb 100644 --- a/src/meta/processors/job/JobManager.cpp +++ b/src/meta/processors/job/JobManager.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "common/stats/StatsManager.h" #include "common/time/WallClock.h" @@ -42,7 +43,10 @@ JobManager* JobManager::getInstance() { return &inst; } -bool JobManager::init(nebula::kvstore::KVStore* store, AdminClient* adminClient) { +bool JobManager::init(nebula::kvstore::KVStore* store, + AdminClient* adminClient, + std::shared_ptr factory) { + executorFactory_ = factory; adminClient_ = adminClient; if (store == nullptr) { return false; @@ -97,7 +101,7 @@ nebula::cpp2::ErrorCode JobManager::handleRemainingJobs() { if (nebula::ok(optJobRet)) { auto optJob = nebula::value(optJobRet); std::unique_ptr je = - JobExecutorFactory::createJobExecutor(optJob, kvStore_, adminClient_); + executorFactory_->createJobExecutor(optJob, kvStore_, adminClient_); // Only balance would change if (optJob.getStatus() == cpp2::JobStatus::RUNNING && je->isMetaJob()) { jds.emplace_back(std::move(optJob)); @@ -235,7 +239,7 @@ folly::Future JobManager::runJobInternal(const JobDescr iter = this->muJobFinished_.emplace(spaceId, std::make_unique()).first; } std::lock_guard lk(*(iter->second)); - auto je = JobExecutorFactory::createJobExecutor(jobDesc, kvStore_, adminClient_); + auto je = executorFactory_->createJobExecutor(jobDesc, kvStore_, adminClient_); jobExec = je.get(); runningJobs_.emplace(jobDesc.getJobId(), std::move(je)); @@ -440,7 +444,7 @@ nebula::cpp2::ErrorCode JobManager::saveTaskStatus(TaskDescription& td, } auto optJobDesc = nebula::value(optJobDescRet); - auto jobExec = JobExecutorFactory::createJobExecutor(optJobDesc, kvStore_, adminClient_); + auto jobExec = executorFactory_->createJobExecutor(optJobDesc, kvStore_, adminClient_); if (!jobExec) { LOG(INFO) << folly::sformat("createJobExecutor failed(), jobId={}", jobId); @@ -682,6 +686,11 @@ bool JobManager::isExpiredJob(JobDescription& jobDesc) { return false; } auto jobStart = jobDesc.getStartTime(); + if (jobStart == 0) { + // should not happend, but just in case keep this job + LOG(INFO) << "Job " << jobDesc.getJobId() << " start time is not set, keep it for now"; + return false; + } auto duration = std::difftime(nebula::time::WallClock::fastNowInSec(), jobStart); return duration > FLAGS_job_expired_secs; } @@ -848,7 +857,9 @@ ErrorOr JobManager::recoverJob( for (auto& [id, job] : allJobs) { auto status = job.getStatus(); if (status == cpp2::JobStatus::FAILED || status == cpp2::JobStatus::STOPPED) { - jobsMaybeRecover.emplace(id); + if (!isExpiredJob(job)) { + jobsMaybeRecover.emplace(id); + } } } std::set::reverse_iterator lastBalaceJobRecoverIt = jobsMaybeRecover.rend(); @@ -869,7 +880,8 @@ ErrorOr JobManager::recoverJob( JobID jid; bool jobExist = checkOnRunningJobExist(spaceId, job.getJobType(), job.getParas(), jid); if (!jobExist) { - job.setStatus(cpp2::JobStatus::QUEUE, true); + job.setStatus(cpp2::JobStatus::QUEUE, true); // which cause the job execute again + job.setErrorCode(nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); auto jobKey = MetaKeyUtils::jobKey(job.getSpace(), jobId); auto jobVal = MetaKeyUtils::jobVal(job.getJobType(), job.getParas(), @@ -919,9 +931,8 @@ ErrorOr JobManager::recoverJob( auto jobType = allJobs[*it].getJobType(); if (jobType == cpp2::JobType::DATA_BALANCE || jobType == cpp2::JobType::ZONE_BALANCE) { if (jobIdSet.empty() || jobIdSet.count(*it)) { - LOG(INFO) << "can't recover a balance job " << *lastBalaceJobRecoverIt - << " when there's a newer balance job " << *lastBalaceJobRecoverIt - << " stopped or failed"; + LOG(INFO) << "can't recover a balance job " << *it << " when there's a newer balance job " + << *lastBalaceJobRecoverIt << " stopped or failed"; } it = jobsMaybeRecover.erase(it); } else { @@ -931,7 +942,7 @@ ErrorOr JobManager::recoverJob( if (*lastBalaceJobRecoverIt < lastBalanceJobFinished) { if (jobIdSet.empty() || jobIdSet.count(*lastBalaceJobRecoverIt)) { LOG(INFO) << "can't recover a balance job " << *lastBalaceJobRecoverIt - << " that before a finished balance job " << lastBalanceJobFinished; + << " when there's a newer balance job " << lastBalanceJobFinished << " finished"; } jobsMaybeRecover.erase(*lastBalaceJobRecoverIt); } diff --git a/src/meta/processors/job/JobManager.h b/src/meta/processors/job/JobManager.h index 1761bc303ae..a3080c92e8f 100644 --- a/src/meta/processors/job/JobManager.h +++ b/src/meta/processors/job/JobManager.h @@ -11,6 +11,7 @@ #include #include +#include #include "common/base/Base.h" #include "common/base/ErrorOr.h" @@ -18,6 +19,7 @@ #include "interface/gen-cpp2/meta_types.h" #include "kvstore/NebulaStore.h" #include "meta/processors/job/JobDescription.h" +#include "meta/processors/job/JobExecutor.h" #include "meta/processors/job/JobStatus.h" #include "meta/processors/job/StorageJobExecutor.h" #include "meta/processors/job/TaskDescription.h" @@ -80,7 +82,9 @@ class JobManager : public boost::noncopyable, public nebula::cpp::NonMovable { * @param adminClient * @return true if the init is successful */ - bool init(nebula::kvstore::KVStore* store, AdminClient* adminClient); + bool init(nebula::kvstore::KVStore* store, + AdminClient* adminClient, + std::shared_ptr factory = std::make_shared()); /** * @brief Called when receive a system signal @@ -331,6 +335,7 @@ class JobManager : public boost::noncopyable, public nebula::cpp::NonMovable { std::atomic status_ = JbmgrStatus::NOT_START; std::unique_ptr executor_; + std::shared_ptr executorFactory_; }; } // namespace meta diff --git a/src/meta/processors/job/JobStatus.cpp b/src/meta/processors/job/JobStatus.cpp index 67b90fe16b4..297273d2635 100644 --- a/src/meta/processors/job/JobStatus.cpp +++ b/src/meta/processors/job/JobStatus.cpp @@ -30,6 +30,10 @@ bool JobStatus::laterThan(Status lhs, Status rhs) { return phaseNumber(lhs) > phaseNumber(rhs); } +bool JobStatus::notSetable(Status st) { + return st == Status::FINISHED; +} + std::string JobStatus::toString(Status st) { switch (st) { case Status::QUEUE: diff --git a/src/meta/processors/job/JobStatus.h b/src/meta/processors/job/JobStatus.h index bb7a173f057..927a3573b1b 100644 --- a/src/meta/processors/job/JobStatus.h +++ b/src/meta/processors/job/JobStatus.h @@ -9,6 +9,7 @@ #include #include +#include "common/base/Status.h" #include "interface/gen-cpp2/meta_types.h" namespace nebula { @@ -20,6 +21,7 @@ class JobStatus { public: static std::string toString(Status st); static bool laterThan(Status lhs, Status rhs); + static bool notSetable(Status st); private: static int phaseNumber(Status st); diff --git a/src/meta/test/JobManagerTest.cpp b/src/meta/test/JobManagerTest.cpp index f1ed97d86ce..d85562df548 100644 --- a/src/meta/test/JobManagerTest.cpp +++ b/src/meta/test/JobManagerTest.cpp @@ -4,22 +4,37 @@ */ #include +#include #include #include +#include + +#include +#include #include "common/base/Base.h" #include "common/fs/TempDir.h" +#include "common/thrift/ThriftTypes.h" +#include "interface/gen-cpp2/common_types.h" +#include "interface/gen-cpp2/meta_types.h" #include "kvstore/Common.h" +#include "kvstore/KVStore.h" #include "meta/ActiveHostsMan.h" #include "meta/processors/job/DownloadJobExecutor.h" #include "meta/processors/job/IngestJobExecutor.h" +#include "meta/processors/job/JobDescription.h" +#include "meta/processors/job/JobExecutor.h" #include "meta/processors/job/JobManager.h" +#include "meta/processors/job/JobStatus.h" #include "meta/processors/job/TaskDescription.h" #include "meta/test/MockAdminClient.h" #include "meta/test/MockHdfsHelper.h" #include "meta/test/TestUtils.h" #include "webservice/WebService.h" +DECLARE_int32(job_check_intervals); +DECLARE_double(job_expired_secs); + namespace nebula { namespace meta { @@ -33,9 +48,145 @@ using ::testing::Return; using ::testing::SetArgPointee; using ::testing::StrictMock; +class MockExecutorFactory : public JobExecutorFactory { + public: + MockExecutorFactory() = default; + ~MockExecutorFactory() override = default; + + MOCK_METHOD(std::unique_ptr, + createJobExecutor, + (const JobDescription& jd, kvstore::KVStore* store, AdminClient* client), + (override)); +}; + +class DummyExecutor : public JobExecutor { + public: + DummyExecutor() = default; + ~DummyExecutor() override = default; + + explicit DummyExecutor(const JobDescription& desc, kvstore::KVStore* store) + : JobExecutor(store, desc.getSpace()), jobDesc_(desc) {} + + nebula::cpp2::ErrorCode check() override { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + nebula::cpp2::ErrorCode prepare() override { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + folly::Future execute() override { + folly::Promise promise; + promise.setValue(nebula::cpp2::ErrorCode::SUCCEEDED); + jobDesc_.setStatus(cpp2::JobStatus::FINISHED, true); + return promise.getFuture(); + } + + nebula::cpp2::ErrorCode stop() override { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + nebula::cpp2::ErrorCode finish(bool ret) override { + UNUSED(ret); + setJobDescStatus(cpp2::JobStatus::FINISHED); + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + nebula::cpp2::ErrorCode recovery() override { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + bool isMetaJob() override = 0; + + JobDescription getJobDescription() override { + return jobDesc_; + } + + nebula::cpp2::ErrorCode saveSpecialTaskStatus(const cpp2::ReportTaskReq&) override { + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + bool isRunning() override { + return isRunning_.load(); + } + + void resetRunningStatus() override { + isRunning_.store(false); + } + + public: + void setJobDescStatus(cpp2::JobStatus status) { + jobDesc_.setStatus(status, true); + } + + protected: + JobDescription jobDesc_; +}; + +class DummyMetaExecutor : public DummyExecutor { + public: + DummyMetaExecutor() = default; + ~DummyMetaExecutor() override = default; + + explicit DummyMetaExecutor(const JobDescription& desc, kvstore::KVStore* store) + : DummyExecutor(desc, store) {} + + bool isMetaJob() override { + return true; + } +}; + +class DummyStorageExecutor : public DummyExecutor { + public: + DummyStorageExecutor() = default; + ~DummyStorageExecutor() override = default; + + explicit DummyStorageExecutor(const JobDescription& desc, kvstore::KVStore* store) + : DummyExecutor(desc, store) {} + + bool isMetaJob() override { + return false; + } + + folly::Future execute() override { + folly::Promise promise; + promise.setValue(nebula::cpp2::ErrorCode::SUCCEEDED); + + TaskDescription task(space_, jobDesc_.getJobId(), 0, HostAddr("", 0)); + auto taskKey = MetaKeyUtils::taskKey(task.getSpace(), task.getJobId(), task.getTaskId()); + auto taskVal = MetaKeyUtils::taskVal(task.getHost(), + task.getStatus(), + task.getStartTime(), + task.getStopTime(), + task.getErrorCode()); + std::vector data; + data.emplace_back(std::move(taskKey), std::move(taskVal)); + folly::Baton baton; + auto rc = nebula::cpp2::ErrorCode::SUCCEEDED; + kvstore_->asyncMultiPut( + kDefaultSpaceId, kDefaultPartId, std::move(data), [&](nebula::cpp2::ErrorCode code) { + rc = code; + baton.post(); + }); + baton.wait(); + return promise.getFuture(); + } +}; + +class UnstoppableDummyStorageExecutor : public DummyStorageExecutor { + public: + UnstoppableDummyStorageExecutor(const JobDescription& desc, kvstore::KVStore* store) + : DummyStorageExecutor(desc, store) {} + nebula::cpp2::ErrorCode stop() override { + return nebula::cpp2::ErrorCode::E_JOB_NOT_STOPPABLE; + } +}; + class JobManagerTest : public ::testing::Test { protected: void SetUp() override { + job_interval = FLAGS_job_check_intervals; // save the original value + FLAGS_job_check_intervals = 50 * 1000; // 50ms for test to control schedule. rootPath_ = std::make_unique("/tmp/JobManager.XXXXXX"); mock::MockCluster cluster; kv_ = cluster.initMetaKV(rootPath_->path()); @@ -69,7 +220,7 @@ class JobManagerTest : public ::testing::Test { [] { return folly::Future>(true); }); } - std::unique_ptr> getJobManager() { + JobManager* initJobManager(std::shared_ptr factory = nullptr) { std::unique_ptr> jobMgr( new JobManager(), [](JobManager* p) { std::tuple opJobId; @@ -80,57 +231,1031 @@ class JobManagerTest : public ::testing::Test { }); jobMgr->status_ = JobManager::JbmgrStatus::NOT_START; jobMgr->kvStore_ = kv_.get(); - jobMgr->init(kv_.get(), adminClient_.get()); - return jobMgr; + if (factory == nullptr) { + jobMgr->init(kv_.get(), adminClient_.get()); + } else { + jobMgr->init(kv_.get(), adminClient_.get(), factory); + } + jobMgr_ = std::move(jobMgr); + return jobMgr_.get(); + } + + void disableSchedule(JobManager* jobMgr) { + jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; + jobMgr->bgThread_.join(); + } + + void enableSchedule(JobManager* jobMgr) { + jobMgr->status_.store(JobManager::JbmgrStatus::IDLE, std::memory_order_release); + jobMgr->bgThread_ = std::thread(&JobManager::scheduleThread, jobMgr); + } + + void reportTaskFinish(nebula::cpp2::ErrorCode code, + GraphSpaceID spaceId, + int32_t jobId, + int32_t taskId) { + cpp2::ReportTaskReq req; + req.code_ref() = code; + req.space_id_ref() = spaceId; + req.job_id_ref() = jobId; + req.task_id_ref() = taskId; + jobMgr_->reportTaskFinish(req); + } + + nebula::cpp2::ErrorCode save(const std::string& k, const std::string& v) { + return jobMgr_->save(k, v); } void TearDown() override { + FLAGS_job_check_intervals = job_interval; // restore the original value + if (jobMgr_) { + if (jobMgr_->status_ != JobManager::JbmgrStatus::STOPPED) { + jobMgr_->shutDown(); + } + jobMgr_.reset(); + } + kv_.reset(); rootPath_.reset(); } + int32_t job_interval{0}; std::unique_ptr rootPath_{nullptr}; std::unique_ptr kv_{nullptr}; std::unique_ptr adminClient_{nullptr}; + std::unique_ptr> jobMgr_{nullptr}; }; HostAddr toHost(std::string strIp) { return HostAddr(strIp, 0); } -TEST_F(JobManagerTest, AddJob) { - std::unique_ptr> jobMgr = getJobManager(); - // For preventing job schedule in JobManager - jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; - jobMgr->bgThread_.join(); +TEST_F(JobManagerTest, AddAndSchedule) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a job GraphSpaceID spaceId = 1; - JobID jobId = 2; - JobDescription jobDesc(spaceId, jobId, cpp2::JobType::COMPACT); - auto rc = jobMgr->addJob(jobDesc); - ASSERT_EQ(rc, nebula::cpp2::ErrorCode::SUCCEEDED); + JobID jobId = 1; + std::vector paras{"tag_index_name"}; - // If there is a failed data balance job, a new job cannot be added - JobID jobId3 = 3; - JobDescription jobDesc3(spaceId, jobId3, cpp2::JobType::DATA_BALANCE); - jobDesc3.setStatus(cpp2::JobStatus::FAILED); - auto jobKey = MetaKeyUtils::jobKey(jobDesc3.getSpace(), jobDesc3.getJobId()); - auto jobVal = MetaKeyUtils::jobVal(jobDesc3.getJobType(), - jobDesc3.getParas(), - jobDesc3.getStatus(), - jobDesc3.getStartTime(), - jobDesc3.getStopTime(), - jobDesc3.getErrorCode()); - jobMgr->save(std::move(jobKey), std::move(jobVal)); + JobDescription desc{spaceId, jobId, cpp2::JobType::REBUILD_TAG_INDEX, paras}; + disableSchedule(jobMgr); + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // Add job with different paras is okay for job not data/zone balance. + JobID newJobId = 2; + JobDescription desc2{spaceId, newJobId, cpp2::JobType::REBUILD_TAG_INDEX}; + auto runningJobExist = + jobMgr->checkOnRunningJobExist(spaceId, desc2.getJobType(), desc2.getParas(), newJobId); + ASSERT_FALSE(runningJobExist); + } + + { + // Add job with same paras is not okay for job not data/zone balance. + JobID newJobId = 2; + JobDescription desc2{spaceId, newJobId, cpp2::JobType::REBUILD_TAG_INDEX, paras}; + auto runningJobExist = + jobMgr->checkOnRunningJobExist(spaceId, desc2.getJobType(), desc2.getParas(), newJobId); + ASSERT_TRUE(runningJobExist); + } + + { + // Job should not affect other space. + JobID otherSpaceJobId = 3; + GraphSpaceID otherSpaceId = 2; + JobDescription desc3{otherSpaceId, otherSpaceJobId, cpp2::JobType::REBUILD_TAG_INDEX}; + auto otherRunningJobExist = jobMgr->checkOnRunningJobExist( + otherSpaceId, desc3.getJobType(), desc3.getParas(), otherSpaceJobId); + ASSERT_FALSE(otherRunningJobExist); + } + + { + // Test show jobs when job is still in queue + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + + // Test job in queue can not be recovered. + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 0); + } + { + enableSchedule(jobMgr); + // Wait schedule thread to schedule the job + usleep(FLAGS_job_check_intervals * 2); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::RUNNING); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + + // Test job running can not be recovered. + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 0); + } + { + // report task finish. + reportTaskFinish(nebula::cpp2::ErrorCode::SUCCEEDED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + + // Check job is finished. + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + + // Test job finished can not be recovered. + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 0); + } +} + +class DummyFailedDataBalanceExecutor : public DummyMetaExecutor { + public: + explicit DummyFailedDataBalanceExecutor(const JobDescription& desc, kvstore::KVStore* store) + : DummyMetaExecutor(desc, store) {} + + folly::Future execute() override { + folly::Promise promise; + promise.setValue(nebula::cpp2::ErrorCode::E_BALANCER_FAILURE); + setJobDescStatus(cpp2::JobStatus::FAILED); + return promise.getFuture(); + } +}; + +TEST_F(JobManagerTest, FailedDataBalanceBlockFollowing) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a failed data balance job + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription desc{spaceId, jobId, cpp2::JobType::DATA_BALANCE}; + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { // wait schedule thread to schedule the job + usleep(FLAGS_job_check_intervals * 2); + } + { + // make sure job is failed + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FAILED); + EXPECT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_BALANCER_FAILURE); + } + { + // Check new job should be blocked + JobID newJobId = 2; + JobDescription desc2{spaceId, newJobId, cpp2::JobType::DATA_BALANCE}; + auto rc = jobMgr->checkNeedRecoverJobExist(spaceId, desc2.getJobType()); + ASSERT_EQ(rc, nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER); + } +} + +TEST_F(JobManagerTest, StopAndRecoverQueueJob) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a job + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription desc{spaceId, jobId, cpp2::JobType::STATS}; + disableSchedule(jobMgr); + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // Stop job and check result + ec = jobMgr->stopJob(spaceId, jobId); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::STOPPED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + // Note: there's 1 job in queue, but it should be skipped when schedule. + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + + enableSchedule(jobMgr); + usleep(FLAGS_job_check_intervals * 2); + // job is skipped now. since it's stopped. + jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::STOPPED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + disableSchedule(jobMgr); + // recover job and check result + std::vector ids; + ids.emplace_back(jobId); + auto ret = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(ret)); + auto recoverNum = nebula::value(ret); + ASSERT_EQ(recoverNum, 1); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + enableSchedule(jobMgr); + usleep(FLAGS_job_check_intervals * 2); + // running recoveryed job and return success + reportTaskFinish(nebula::cpp2::ErrorCode::SUCCEEDED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + } + + { + // recovered job will succeed + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + } +} + +TEST_F(JobManagerTest, StopAndRecoverRunningJob) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a job + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription desc{spaceId, jobId, cpp2::JobType::STATS}; + + disableSchedule(jobMgr); + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // Check job in queue + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } - rc = jobMgr->checkNeedRecoverJobExist(spaceId, jobDesc3.getJobType()); - ASSERT_EQ(rc, nebula::cpp2::ErrorCode::E_JOB_NEED_RECOVER); + { + enableSchedule(jobMgr); + // Wait job running and then stop + usleep(FLAGS_job_check_intervals * 2); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::RUNNING); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + // stop running job and check result + ec = jobMgr->stopJob(spaceId, jobId); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + // Wait job stopped and then recover + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::STOPPED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + // running job will report user cancel and finally failed. + reportTaskFinish(nebula::cpp2::ErrorCode::E_USER_CANCEL, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_USER_CANCEL); + } + + { + // recover failed job and check result + std::vector ids; + ids.emplace_back(jobId); + disableSchedule(jobMgr); + auto ret = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(ret)); + auto recoverNum = nebula::value(ret); + ASSERT_EQ(recoverNum, 1); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + enableSchedule(jobMgr); + // wait runnning and then report success + usleep(FLAGS_job_check_intervals * 2); + reportTaskFinish(nebula::cpp2::ErrorCode::SUCCEEDED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + // recover job will succeed + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + } +} + +TEST_F(JobManagerTest, StopAndRecoverUnstoppableJob) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a job + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription desc{spaceId, jobId, cpp2::JobType::COMPACT}; + disableSchedule(jobMgr); + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // Check job is in queue + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + + { + // Stop job will succeed if job is in queue + ec = jobMgr->stopJob(spaceId, jobId); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + // Check job is stopped + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 1); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::STOPPED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + } + { + // recover job will succeed + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 1); + + // Check job is in queue + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 2); // queue 1 add and 1 recover + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + // a recover job can not be stopped? + } + { + enableSchedule(jobMgr); + // wait job running and stop it + usleep(FLAGS_job_check_intervals * 2); + + auto jobSize = jobMgr->jobSize(); + EXPECT_EQ(jobSize, 1); // a running job and a queue job(recover) + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::RUNNING); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + + // stop a running unstoppable job will fail + ec = jobMgr->stopJob(spaceId, jobId); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::E_JOB_NOT_STOPPABLE); + } + { + // wait job finish + reportTaskFinish(nebula::cpp2::ErrorCode::SUCCEEDED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + // BUG: This job is finished once, and recover job is running again and skipped(fixed) + // BUG: The order is not correct. + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + } + { + ec = jobMgr->stopJob(spaceId, jobId); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::E_JOB_ALREADY_FINISH); + } +} + +TEST_F(JobManagerTest, RecoverRunningFailedJob) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a job + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription desc{spaceId, jobId, cpp2::JobType::STATS}; + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // Check job running and failed + usleep(FLAGS_job_check_intervals * 2); + reportTaskFinish(nebula::cpp2::ErrorCode::E_TASK_EXECUTION_FAILED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_TASK_EXECUTION_FAILED); + } + { + // recover job will succeed + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 1); + + usleep(FLAGS_job_check_intervals * 2); + reportTaskFinish(nebula::cpp2::ErrorCode::SUCCEEDED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + } +} + +TEST_F(JobManagerTest, RecoverBalanceJob) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + // Add a job + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription desc{spaceId, jobId, cpp2::JobType::ZONE_BALANCE}; + auto ec = jobMgr->addJob(desc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // check job failed + usleep(FLAGS_job_check_intervals * 2); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_BALANCER_FAILURE); + } + + // add another job and show failed. + JobID newJobId = 2; + JobDescription newDesc{spaceId, newJobId, cpp2::JobType::ZONE_BALANCE}; + ec = jobMgr->addJob(newDesc); + ASSERT_EQ(ec, nebula::cpp2::ErrorCode::SUCCEEDED); + + { + // check job failed + usleep(FLAGS_job_check_intervals * 2); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 2); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::E_BALANCER_FAILURE); + ASSERT_EQ(showJobs[1].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[1].get_code(), nebula::cpp2::ErrorCode::E_BALANCER_FAILURE); + } + // new job should succeed. + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + + { + // recover first job should failed. + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 0); + } + { + // recover new job should ok. + std::vector ids; + ids.emplace_back(newJobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 1); + } + { + // recover job should finish. + usleep(FLAGS_job_check_intervals * 2); + auto jobSize = jobMgr->jobSize(); + ASSERT_EQ(jobSize, 0); + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 2); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + ASSERT_EQ(showJobs[1].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[1].get_code(), nebula::cpp2::ErrorCode::E_BALANCER_FAILURE); + } + { + // recover first job now should failed since a new finished balance job exists. + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 0); + } +} + +TEST_F(JobManagerTest, RecoverExpiredJobTest) { + auto factory = std::make_shared(); + EXPECT_CALL(*factory, createJobExecutor(_, _, _)) + .WillRepeatedly([](const JobDescription& jd, kvstore::KVStore* store, AdminClient* client) { + UNUSED(client); + return std::make_unique(jd, store); + }); + auto jobMgr = initJobManager(factory); + + auto nowTimeInSec = nebula::time::WallClock::fastNowInSec(); + auto expiredJobTime = std::difftime(nowTimeInSec, FLAGS_job_expired_secs + 1); + { + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::FAILED, + expiredJobTime, + expiredJobTime + 2, + nebula::cpp2::ErrorCode::E_UNKNOWN); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + JobID jobId = 1; + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 0); + } + // recover job should finish. + { + GraphSpaceID spaceId = 1; + JobID jobId = 2; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::FAILED, + expiredJobTime + 10, + expiredJobTime + 11, + nebula::cpp2::ErrorCode::E_UNKNOWN); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + JobID jobId = 2; + std::vector ids; + ids.emplace_back(jobId); + auto recoverRet = jobMgr->recoverJob(spaceId, ids); + ASSERT_TRUE(nebula::ok(recoverRet)); + auto recoverNum = nebula::value(recoverRet); + ASSERT_EQ(recoverNum, 1); + + usleep(FLAGS_job_check_intervals * 2); + reportTaskFinish(nebula::cpp2::ErrorCode::SUCCEEDED, spaceId, jobId, 0); + usleep(FLAGS_job_check_intervals * 2); + + // should be ok + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_code(), nebula::cpp2::ErrorCode::SUCCEEDED); + } +} + +TEST_F(JobManagerTest, ExpiredJobTest) { + auto jobMgr = initJobManager(); + // For preventing job schedule in JobManager + auto nowTimeInSec = nebula::time::WallClock::fastNowInSec(); + auto expiredJobTime = std::difftime(nowTimeInSec, FLAGS_job_expired_secs + 1); + + // 1. failed job should be expired. + { + GraphSpaceID spaceId = 1; + JobID jobId = 1; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::FAILED, + expiredJobTime, + expiredJobTime + 2, + nebula::cpp2::ErrorCode::E_UNKNOWN); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 0); + } + // 2. finished job should be expired. + { + GraphSpaceID spaceId = 1; + JobID jobId = 2; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::FINISHED, + expiredJobTime, + expiredJobTime + 2, + nebula::cpp2::ErrorCode::SUCCEEDED); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 0); + } + // 3. stopped job should expired. + { + GraphSpaceID spaceId = 1; + JobID jobId = 3; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::STOPPED, + expiredJobTime, + expiredJobTime + 2, + nebula::cpp2::ErrorCode::E_USER_CANCEL); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 0); + } + + // 4. failed job not expired. + { + GraphSpaceID spaceId = 1; + JobID jobId = 4; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::FAILED, + expiredJobTime + 10, + expiredJobTime + 11, + nebula::cpp2::ErrorCode::E_UNKNOWN); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 1); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[0].get_job_id(), 4); + } + // 5. stop job not expired. + { + GraphSpaceID spaceId = 1; + JobID jobId = 5; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::STOPPED, + expiredJobTime + 10, + expiredJobTime + 11, + nebula::cpp2::ErrorCode::E_USER_CANCEL); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 2); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::STOPPED); + ASSERT_EQ(showJobs[0].get_job_id(), 5); + ASSERT_EQ(showJobs[1].get_status(), cpp2::JobStatus::FAILED); + ASSERT_EQ(showJobs[1].get_job_id(), 4); + } + // 6. finished job not expired. + { + GraphSpaceID spaceId = 1; + JobID jobId = 6; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::FINISHED, + expiredJobTime + 10, + expiredJobTime + 11, + nebula::cpp2::ErrorCode::SUCCEEDED); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 3); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::FINISHED); + ASSERT_EQ(showJobs[0].get_job_id(), 6); + } + + // 7. queue job should stay even timeout + { + GraphSpaceID spaceId = 1; + JobID jobId = 7; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::QUEUE, + expiredJobTime, + expiredJobTime + 2, + nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 4); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::QUEUE); + ASSERT_EQ(showJobs[0].get_job_id(), 7); + } + // 8. running job should stay even timeout + { + GraphSpaceID spaceId = 1; + JobID jobId = 8; + JobDescription jobDesc(spaceId, + jobId, + cpp2::JobType::STATS, + {}, + cpp2::JobStatus::RUNNING, + expiredJobTime, + expiredJobTime + 2, + nebula::cpp2::ErrorCode::E_JOB_SUBMITTED); + + auto jobKey = MetaKeyUtils::jobKey(jobDesc.getSpace(), jobDesc.getJobId()); + auto jobVal = MetaKeyUtils::jobVal(jobDesc.getJobType(), + jobDesc.getParas(), + jobDesc.getStatus(), + jobDesc.getStartTime(), + jobDesc.getStopTime(), + jobDesc.getErrorCode()); + save(std::move(jobKey), std::move(jobVal)); + } + { + GraphSpaceID spaceId = 1; + auto showRet = jobMgr->showJobs(spaceId); + ASSERT_TRUE(nebula::ok(showRet)); + auto showJobs = nebula::value(showRet); + ASSERT_EQ(showJobs.size(), 5); + ASSERT_EQ(showJobs[0].get_status(), cpp2::JobStatus::RUNNING); + ASSERT_EQ(showJobs[0].get_job_id(), 8); + } } TEST_F(JobManagerTest, AddRebuildTagIndexJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); // For preventing job schedule in JobManager - jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; - jobMgr->bgThread_.join(); + disableSchedule(jobMgr); std::vector paras{"tag_index_name"}; GraphSpaceID spaceId = 1; JobID jobId = 11; @@ -142,7 +1267,7 @@ TEST_F(JobManagerTest, AddRebuildTagIndexJob) { } TEST_F(JobManagerTest, AddRebuildEdgeIndexJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); // For preventing job schedule in JobManager jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; jobMgr->bgThread_.join(); @@ -206,7 +1331,7 @@ TEST_F(JobManagerTest, IngestJob) { } TEST_F(JobManagerTest, StatsJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); // For preventing job schedule in JobManager jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; jobMgr->bgThread_.join(); @@ -238,7 +1363,7 @@ TEST_F(JobManagerTest, StatsJob) { // Jobs are parallelized between spaces, and serialized by priority within spaces TEST_F(JobManagerTest, JobPriority) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); // For preventing job schedule in JobManager jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; jobMgr->bgThread_.join(); @@ -298,7 +1423,7 @@ TEST_F(JobManagerTest, JobPriority) { } TEST_F(JobManagerTest, JobDeduplication) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); // For preventing job schedule in JobManager jobMgr->status_ = JobManager::JbmgrStatus::STOPPED; jobMgr->bgThread_.join(); @@ -354,7 +1479,7 @@ TEST_F(JobManagerTest, JobDeduplication) { } TEST_F(JobManagerTest, LoadJobDescription) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId = 1; JobID jobId1 = 1; JobDescription jobDesc1(spaceId, jobId1, cpp2::JobType::COMPACT); @@ -381,7 +1506,7 @@ TEST_F(JobManagerTest, LoadJobDescription) { } TEST_F(JobManagerTest, ShowJobs) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId = 1; JobID jobId1 = 1; JobDescription jobDesc1(spaceId, jobId1, cpp2::JobType::COMPACT); @@ -418,7 +1543,7 @@ TEST_F(JobManagerTest, ShowJobs) { } TEST_F(JobManagerTest, ShowJobsFromMultiSpace) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId1 = 1; JobID jobId1 = 1; JobDescription jd1(spaceId1, jobId1, cpp2::JobType::COMPACT); @@ -450,7 +1575,7 @@ TEST_F(JobManagerTest, ShowJobsFromMultiSpace) { } TEST_F(JobManagerTest, ShowJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId = 1; JobID jobId1 = 1; JobDescription jd(spaceId, jobId1, cpp2::JobType::COMPACT); @@ -516,7 +1641,7 @@ TEST_F(JobManagerTest, ShowJob) { } TEST_F(JobManagerTest, ShowJobInOtherSpace) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId1 = 1; JobID jobId1 = 1; JobDescription jd(spaceId1, jobId1, cpp2::JobType::COMPACT); @@ -555,7 +1680,7 @@ TEST_F(JobManagerTest, ShowJobInOtherSpace) { } TEST_F(JobManagerTest, RecoverJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); // set status to prevent running the job since AdminClient is a injector jobMgr->status_.store(JobManager::JbmgrStatus::STOPPED, std::memory_order_release); jobMgr->bgThread_.join(); @@ -733,7 +1858,7 @@ TEST_F(JobManagerTest, RecoverJob) { } TEST_F(JobManagerTest, NotStoppableJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId = 1; PartitionID partId = 1; JobID jobId = 1; @@ -811,7 +1936,7 @@ TEST_F(JobManagerTest, NotStoppableJob) { } TEST_F(JobManagerTest, StoppableJob) { - std::unique_ptr> jobMgr = getJobManager(); + auto jobMgr = initJobManager(); GraphSpaceID spaceId = 1; JobID jobId = 1; From ee090f897a5a85683d59b83b2f48bb6344a0a541 Mon Sep 17 00:00:00 2001 From: Cheng Xuntao <7731943+xtcyclist@users.noreply.github.com> Date: Fri, 6 Jan 2023 11:08:23 +0800 Subject: [PATCH 14/51] Add more match test cases on paths. (#5189) --- tests/tck/features/match/Path.feature | 420 +++++++++++++++++++++++++- 1 file changed, 418 insertions(+), 2 deletions(-) diff --git a/tests/tck/features/match/Path.feature b/tests/tck/features/match/Path.feature index 80f9fb698ee..9f31eca21e5 100644 --- a/tests/tck/features/match/Path.feature +++ b/tests/tck/features/match/Path.feature @@ -38,10 +38,296 @@ Feature: Matching paths | count(p) | count(p2) | | 966 | 966 | + Scenario: overlapping aliases variables + When executing query: + """ + match p = (v1:Label_3)-[e1]-(v1) + return count(p) + """ + Then the result should be, in any order: + | count(p) | + | 28 | + When executing query: + """ + match p2 = (v2:Label_5)-[e2]-(v2) + return count(p2) + """ + Then the result should be, in any order: + | count(p2) | + | 43 | + When executing query: + """ + match p = (v1:Label_3)-[e1]-(v1) + match p2 = (v2:Label_5)-[e2]-(v2) + return count(p), count(p2), count(*) + """ + Then the result should be, in any order: + | count(p) | count(p2) | count(*) | + | 1204 | 1204 | 1204 | + When executing query: + """ + match p = (v1:Label_3)-[e1:Rel_0]-(v1) + match p2 = (v1:Label_3)-[e2:Rel_1]-(v2) + return distinct id(v1) + """ + Then the result should be, in any order: + | id(v1) | + | 57 | + | 47 | + | 97 | + When executing query: + """ + match p = (v1:Label_3)-[e1:Rel_0]-(v1) return distinct id(v1) + """ + Then the result should be, in any order: + | id(v1) | + | 57 | + | 47 | + | 97 | + When executing query: + """ + match p2 = (v1:Label_3)-[e2:Rel_1]-(v2) + where id(v1) in [57, 47, 97] + with v1 as v2 + return id(v2) + """ + Then the result should be, in any order: + | id(v2) | + | 57 | + | 57 | + | 57 | + | 57 | + | 47 | + | 47 | + | 47 | + | 47 | + | 47 | + | 47 | + | 97 | + | 97 | + | 97 | + | 97 | + When executing query: + """ + match p = (v1:Label_3)-[e1:Rel_0]-(v1) + match p2 = (v1:Label_3)-[e2:Rel_1]-(v2) + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 14 | + When executing query: + """ + match p = (v1:Label_3)-[e1:Rel_0]-(v1) + match p2 = (v1:Label_3)-[e1:Rel_0]-(v2) + where p != p2 + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 0 | + When executing query: + """ + match p = (v1:Label_3)-[e1:Rel_0]-(v1) + match p2 = (v1:Label_3)-[e1:Rel_0]-(v2) + where p == p2 + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 3 | + + Scenario: many paths + When executing query: + """ + match p = (v1:Label_3)-[e1:Rel_0]-(v1) + match p2 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p3 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p4 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p5 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p6 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p7 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p8 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p9 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p10 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p11 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p12 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p13 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p14 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p15 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p16 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p17 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p18 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p19 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p20 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p21 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p22 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p23 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p24 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p25 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p26 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p27 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p28 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p29 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p30 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p31 = (v1:Label_3)-[e1:Rel_0]-(v1) + match p32 = (v1:Label_3)-[e1:Rel_0]-(v1) + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 3 | + + Scenario: single vertex + When executing query: + """ + match p = (v) + where id(v) == 1 + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 1 | + When executing query: + """ + match p = (v:Label_3) + return count(p) + """ + Then the result should be, in any order: + | count(p) | + | 59 | + When executing query: + """ + match p = (v:Label_0) + return count(p) + """ + Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. + + @skip #bug to fix: https://github.com/vesoft-inc/nebula/issues/5185 + Scenario: conflicting type + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]-(v1:Label_1), + p2 = (p)-[e2:Rel_2]-(v3:Label_3) + where id(v) in [100] and id(v3) in [80] + return count(p), count(p2) + """ + Then a SemanticError should be raised at runtime: `p': defined with conflicting type + + Scenario: use of defined vertices + # edges cannot be redefined, tested in Scenario: distinct edges and paths + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]->(v1:Label_1), + p2 = (v)-[e1:Rel_0]->(v1) + where p!=p2 + return count(p), count(p2) + """ + Then the result should be, in any order: + | count(p) | count(p2) | + | 0 | 0 | + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]->(v1:Label_1), + p2 = (v)-[e1:Rel_0]->(v1) + where p==p2 + return count(p), count(p2) + """ + Then the result should be, in any order: + | count(p) | count(p2) | + | 94 | 94 | + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]->(v1:Label_1) + match p2 = (v)-[e1:Rel_0]->(v1) + where p!=p2 + return count(p), count(p2) + """ + Then the result should be, in any order: + | count(p) | count(p2) | + | 0 | 0 | + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]->(v1:Label_1) + match p2 = (v)-[e1:Rel_0]->(v1) + where p==p2 + return count(p), count(p2) + """ + Then the result should be, in any order: + | count(p) | count(p2) | + | 94 | 94 | + + @skip #bug to fix: https://github.com/vesoft-inc/nebula/issues/5187 + Scenario: edge directions in paths + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]->(v1:Label_1), + p2 = (v)-[e1:Rel_0]->(v1) + where id(v) == 47 + and p != p2 + and e == e1 + and v == v + and v1 == v1 + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 0 | + + Scenario: use defined path variable + When executing query: + """ + match p = (v:Label_5)-[e:Rel_0]->(v1:Label_1) + return count(nodes(p)), count(keys(v)), count(labels(v)), id(v) + """ + Then the result should be, in any order: + | count(nodes(p)) | count(keys(v)) | count(labels(v)) | id(v) | + | 1 | 1 | 1 | 45 | + | 1 | 1 | 1 | 30 | + | 1 | 1 | 1 | 20 | + | 1 | 1 | 1 | 16 | + | 2 | 2 | 2 | 68 | + | 1 | 1 | 1 | 41 | + | 1 | 1 | 1 | 100 | + | 1 | 1 | 1 | 65 | + | 1 | 1 | 1 | 11 | + | 2 | 2 | 2 | 3 | + | 3 | 3 | 3 | 77 | + | 3 | 3 | 3 | 48 | + | 2 | 2 | 2 | 72 | + | 5 | 5 | 5 | 67 | + | 2 | 2 | 2 | 8 | + | 2 | 2 | 2 | 36 | + | 2 | 2 | 2 | 95 | + | 1 | 1 | 1 | 1 | + | 2 | 2 | 2 | 60 | + | 1 | 1 | 1 | 94 | + | 2 | 2 | 2 | 35 | + | 1 | 1 | 1 | 98 | + | 2 | 2 | 2 | 39 | + | 3 | 3 | 3 | 54 | + | 2 | 2 | 2 | 24 | + | 1 | 1 | 1 | 83 | + | 2 | 2 | 2 | 53 | + | 2 | 2 | 2 | 82 | + | 1 | 1 | 1 | 102 | + | 3 | 3 | 3 | 27 | + | 1 | 1 | 1 | 56 | + | 1 | 1 | 1 | 76 | + | 2 | 2 | 2 | 47 | + | 3 | 3 | 3 | 59 | + | 24 | 24 | 24 | 383 | + | 1 | 1 | 1 | 88 | + | 2 | 2 | 2 | 97 | + | 2 | 2 | 2 | 38 | + | 1 | 1 | 1 | 96 | + | 3 | 3 | 3 | 99 | + Scenario: symmetry paths When executing query: """ - match p1 = (v1)-[e*1..2]->(v2) where id(v1) in [6] + match p1 = (v1)-[e*1..2]->(v2) + where id(v1) in [6] match p2 = (v2)-[e*1..2]->(v1) return count(*) """ @@ -50,7 +336,8 @@ Feature: Matching paths | 16 | When executing query: """ - match p1 = (v1)-[e*1..2]->(v2) where id(v1) in [6] + match p1 = (v1)-[e*1..2]->(v2) + where id(v1) in [6] match p2 = (v2)-[e*1..2]->(v1) return size(e) """ @@ -72,3 +359,132 @@ Feature: Matching paths | 2 | | 2 | | 2 | + + Scenario: src dst variables in paths + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0]->(v1) + return src(e), dst(e) + """ + Then the result should be, in any order: + | src(e) | dst(e) | + | 97 | 97 | + | 47 | 47 | + | 6 | 6 | + | 79 | 79 | + | 19 | 19 | + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0]->(v2) + where id(v1) == id(v2) + return src(e), dst(e) + """ + Then the result should be, in any order: + | src(e) | dst(e) | + | 97 | 97 | + | 47 | 47 | + | 6 | 6 | + | 79 | 79 | + | 19 | 19 | + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0]->(v2) + where id(v1) != id(v2) + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 185 | + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0]->(v2) + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 190 | + + Scenario: edgelist or single edge in paths + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0*2..4]->(v2) + where id(v1) in [50] + return size(e), count(*) + """ + Then the result should be, in any order: + | size(e) | count(*) | + | 4 | 4 | + | 3 | 2 | + | 2 | 1 | + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0*2..4]->(v2) + where id(v1) in [50] + unwind e as x + return x.Rel_0_7_Double, src(x) + """ + Then the result should be, in any order: + | x.Rel_0_7_Double | src(x) | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.76606 | 39 | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.364565 | 39 | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.76606 | 39 | + | 0.019344 | 12 | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.76606 | 39 | + | 0.258873 | 12 | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.364565 | 39 | + | 0.041292 | 76 | + | 0.917773 | 50 | + | 0.266255 | 93 | + | 0.364565 | 39 | + | 0.893019 | 76 | + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0*2..2]->(v2) + where id(v1) in [50] + unwind e as x + return x.Rel_0_7_Double, src(x) + """ + Then the result should be, in any order: + | x.Rel_0_7_Double | src(x) | + | 0.917773 | 50 | + | 0.266255 | 93 | + When executing query: + """ + match p = (v1:Label_11)-[e:Rel_0*1..1]->(v2) + where id(v1) in [50] + unwind e as x + return x.Rel_0_7_Double, src(x) + """ + Then the result should be, in any order: + | x.Rel_0_7_Double | src(x) | + | 0.917773 | 50 | + When executing query: + """ + match (v1)-[e*1..2]->(v2) + where id(v1) in [6] + match (v2)-[e*1..2]->(v1) + return count(*) + """ + Then the result should be, in any order: + | count(*) | + | 16 | + When executing query: + """ + match (v1)-[e*1..2]->(v2) + where id(v1) in [6] + match (v2)-[e*1..1]->(v1) + return count(*) + """ + Then a SemanticError should be raised at runtime: e binding to different type: Edge vs EdgeList From 7b2e2a0b485c892479fa238b1f93c536e36cf821 Mon Sep 17 00:00:00 2001 From: codesigner Date: Fri, 6 Jan 2023 15:38:41 +0800 Subject: [PATCH 15/51] improve memtracker, add missed check & remove unnecessary thenError&tryCatch check (#5199) * [memtracker] check code run with memoery check on all works refine code all code memory checked fix lint refine code & fix build with gcc+sanitize * fix build break * fix lint * refine code * remove debug code * fix test fail build with debug * fix test fail build with debug * restore commented test * minor * minor --- cmake/nebula/SanitizerConfig.cmake | 1 + src/clients/storage/StorageClientBase-inl.h | 50 ++-- src/codec/test/CMakeLists.txt | 2 + src/common/base/Status.cpp | 4 + src/common/base/Status.h | 5 + src/common/base/test/CMakeLists.txt | 2 + src/common/datatypes/DataSetOps-inl.h | 13 + src/common/datatypes/ValueOps-inl.h | 1 + src/common/datatypes/test/CMakeLists.txt | 2 + src/common/expression/test/CMakeLists.txt | 2 + src/common/geo/test/CMakeLists.txt | 3 + src/common/graph/tests/CMakeLists.txt | 3 + src/common/id/test/CMakeLists.txt | 2 + src/common/memory/MemoryTracker.cpp | 4 + src/common/memory/MemoryTracker.h | 8 + src/common/memory/MemoryUtils.cpp | 16 +- src/common/memory/NewDelete.cpp | 13 +- src/common/utils/test/CMakeLists.txt | 11 + src/graph/executor/Executor.cpp | 3 + src/graph/executor/Executor.h | 17 +- src/graph/executor/StorageAccessExecutor.h | 6 +- .../executor/algo/BFSShortestPathExecutor.cpp | 11 +- src/graph/executor/algo/BatchShortestPath.cpp | 170 ++++++------ .../algo/MultiShortestPathExecutor.cpp | 42 ++- .../executor/algo/ProduceAllPathsExecutor.cpp | 12 +- src/graph/executor/algo/ShortestPathBase.cpp | 7 + src/graph/executor/algo/ShortestPathBase.h | 2 + .../executor/algo/ShortestPathExecutor.cpp | 2 + .../executor/algo/SingleShortestPath.cpp | 112 +++----- src/graph/executor/algo/SubgraphExecutor.cpp | 7 +- src/graph/executor/logic/ArgumentExecutor.cpp | 1 + src/graph/executor/maintain/EdgeExecutor.cpp | 39 +-- .../executor/maintain/EdgeIndexExecutor.cpp | 37 +-- .../executor/maintain/FTIndexExecutor.cpp | 98 +++---- src/graph/executor/maintain/TagExecutor.cpp | 43 +-- .../executor/maintain/TagIndexExecutor.cpp | 50 +--- src/graph/executor/mutate/DeleteExecutor.cpp | 18 -- src/graph/executor/mutate/InsertExecutor.cpp | 12 - src/graph/executor/mutate/UpdateExecutor.cpp | 12 - .../executor/query/AggregateExecutor.cpp | 1 + .../executor/query/AppendVerticesExecutor.cpp | 7 +- src/graph/executor/query/DedupExecutor.cpp | 2 + src/graph/executor/query/FilterExecutor.cpp | 3 + .../executor/query/GetDstBySrcExecutor.cpp | 6 - src/graph/executor/query/GetEdgesExecutor.cpp | 6 - .../executor/query/GetNeighborsExecutor.cpp | 6 - .../executor/query/GetVerticesExecutor.cpp | 6 - .../executor/query/IndexScanExecutor.cpp | 7 +- src/graph/executor/query/ProjectExecutor.cpp | 1 + .../executor/query/ScanEdgesExecutor.cpp | 6 - .../executor/query/ScanVerticesExecutor.cpp | 6 - src/graph/executor/query/TraverseExecutor.cpp | 12 +- .../AsyncMsgNotifyBasedScheduler.cpp | 82 ++---- src/graph/service/QueryInstance.cpp | 5 +- src/kvstore/listener/test/CMakeLists.txt | 1 + src/kvstore/raftex/test/CMakeLists.txt | 1 + src/kvstore/test/CMakeLists.txt | 1 + src/kvstore/wal/test/CMakeLists.txt | 1 + src/storage/BaseProcessor.h | 31 ++- src/storage/GraphStorageServiceHandler.cpp | 17 +- src/storage/index/LookupProcessor.cpp | 19 +- src/storage/kv/GetProcessor.cpp | 75 +++-- src/storage/kv/PutProcessor.cpp | 34 +-- src/storage/kv/RemoveProcessor.cpp | 34 +-- src/storage/mutate/AddEdgesProcessor.cpp | 72 +++-- src/storage/mutate/AddVerticesProcessor.cpp | 73 +++-- src/storage/mutate/DeleteEdgesProcessor.cpp | 261 +++++++++--------- src/storage/mutate/DeleteTagsProcessor.cpp | 133 ++++----- .../mutate/DeleteVerticesProcessor.cpp | 168 ++++++----- src/storage/mutate/UpdateEdgeProcessor.cpp | 22 +- src/storage/mutate/UpdateVertexProcessor.cpp | 22 +- src/storage/query/GetDstBySrcProcessor.cpp | 19 +- src/storage/query/GetNeighborsProcessor.cpp | 19 +- src/storage/query/GetPropProcessor.cpp | 19 +- src/storage/query/ScanEdgeProcessor.cpp | 19 +- src/storage/query/ScanVertexProcessor.cpp | 19 +- 76 files changed, 845 insertions(+), 1214 deletions(-) diff --git a/cmake/nebula/SanitizerConfig.cmake b/cmake/nebula/SanitizerConfig.cmake index 28b5d500ef4..9f610e819a3 100644 --- a/cmake/nebula/SanitizerConfig.cmake +++ b/cmake/nebula/SanitizerConfig.cmake @@ -15,6 +15,7 @@ if(ENABLE_ASAN) add_compile_options(-fsanitize=address) add_compile_options(-g) add_compile_options(-fno-omit-frame-pointer) + add_definitions(-DENABLE_ASAN) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") endif() diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h index 84fef19a840..beb8d25d958 100644 --- a/src/clients/storage/StorageClientBase-inl.h +++ b/src/clients/storage/StorageClientBase-inl.h @@ -101,16 +101,21 @@ StorageClientBase::collectResponse( return folly::collectAll(respFutures) .deferValue([this, requests = std::move(requests), totalLatencies, hosts]( std::vector>>&& resps) { + // throw in MemoryCheckGuard verified memory::MemoryCheckGuard guard; StorageRpcResponse rpcResp(resps.size()); for (size_t i = 0; i < resps.size(); i++) { auto& host = hosts->at(i); - auto& tryResp = resps[i]; - std::optional errMsg; + folly::Try>& tryResp = resps[i]; if (tryResp.hasException()) { - errMsg = std::string(tryResp.exception().what().c_str()); + std::string errMsg = tryResp.exception().what().toStdString(); + rpcResp.markFailure(); + LOG(ERROR) << "There some RPC errors: " << errMsg; + auto req = requests.at(host); + auto parts = getReqPartsId(req); + rpcResp.appendFailedParts(parts, nebula::cpp2::ErrorCode::E_RPC_FAILURE); } else { - auto status = std::move(tryResp).value(); + StatusOr status = std::move(tryResp).value(); if (status.ok()) { auto resp = std::move(status).value(); auto result = resp.get_result(); @@ -128,17 +133,18 @@ StorageClientBase::collectResponse( // Keep the response rpcResp.addResponse(std::move(resp)); } else { - errMsg = std::move(status).status().message(); + rpcResp.markFailure(); + Status s = std::move(status).status(); + nebula::cpp2::ErrorCode errorCode = + s.code() == Status::Code::kGraphMemoryExceeded + ? nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED + : nebula::cpp2::ErrorCode::E_RPC_FAILURE; + LOG(ERROR) << "There some RPC errors: " << s.message(); + auto req = requests.at(host); + auto parts = getReqPartsId(req); + rpcResp.appendFailedParts(parts, errorCode); } } - - if (errMsg) { - rpcResp.markFailure(); - LOG(ERROR) << "There some RPC errors: " << errMsg.value(); - auto req = requests.at(host); - auto parts = getReqPartsId(req); - rpcResp.appendFailedParts(parts, nebula::cpp2::ErrorCode::E_RPC_FAILURE); - } } return rpcResp; @@ -160,12 +166,16 @@ folly::Future> StorageClientBaseclient(host, evb, false, FLAGS_storage_client_timeout_ms); + // Encoding invoke Cpp2Ops::write the request to protocol is in current thread, + // do not need to turn on in Cpp2Ops::write return remoteFunc(client.get(), request); }) .thenValue([spaceId, this](Response&& resp) mutable -> StatusOr { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; auto& result = resp.get_result(); for (auto& part : result.get_failed_parts()) { @@ -196,14 +206,12 @@ folly::Future> StorageClientBase{}, - [](const std::bad_alloc&) { - return folly::makeFuture>(std::bad_alloc()); - }) - .thenError(folly::tag_t{}, - [](const std::exception& e) { - return folly::makeFuture>(std::runtime_error(e.what())); - }) + .thenError( + folly::tag_t{}, + [](const std::bad_alloc&) { + return folly::makeFuture>(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + }) .thenError([request, host, spaceId, this]( folly::exception_wrapper&& exWrapper) mutable -> StatusOr { stats::StatsManager::addValue(kNumRpcSentToStoragedFailed); diff --git a/src/codec/test/CMakeLists.txt b/src/codec/test/CMakeLists.txt index 9b4e1272096..8c511451eaf 100644 --- a/src/codec/test/CMakeLists.txt +++ b/src/codec/test/CMakeLists.txt @@ -22,6 +22,8 @@ set(CODEC_TEST_LIBS $ $ $ + $ + $ $ $ $ diff --git a/src/common/base/Status.cpp b/src/common/base/Status.cpp index d80c31cf370..51d4f8698ac 100644 --- a/src/common/base/Status.cpp +++ b/src/common/base/Status.cpp @@ -67,10 +67,14 @@ const char *Status::toString(Code code) { return "StatementEmpty: "; case kSemanticError: return "SemanticError: "; + case kGraphMemoryExceeded: + return "GraphMemoryExceeded: "; case kKeyNotFound: return "KeyNotFound: "; case kPartialSuccess: return "PartialSuccess: "; + case kStorageMemoryExceeded: + return "StorageMemoryExceeded: "; case kSpaceNotFound: return "SpaceNotFound: "; case kHostNotFound: diff --git a/src/common/base/Status.h b/src/common/base/Status.h index 2783cb7a70b..598320ad675 100644 --- a/src/common/base/Status.h +++ b/src/common/base/Status.h @@ -119,12 +119,15 @@ class Status final { // Graph engine errors STATUS_GENERATOR(SyntaxError); STATUS_GENERATOR(SemanticError); + STATUS_GENERATOR(GraphMemoryExceeded); + // Nothing is executed When command is comment STATUS_GENERATOR(StatementEmpty); // Storage engine errors STATUS_GENERATOR(KeyNotFound); STATUS_GENERATOR(PartialSuccess); + STATUS_GENERATOR(StorageMemoryExceeded); // Meta engine errors // TODO(dangleptr) we could use ErrorOr to replace SpaceNotFound here. @@ -166,9 +169,11 @@ class Status final { kSyntaxError = 201, kStatementEmpty = 202, kSemanticError = 203, + kGraphMemoryExceeded = 204, // 3xx, for storage engine errors kKeyNotFound = 301, kPartialSuccess = 302, + kStorageMemoryExceeded = 303, // 4xx, for meta service errors kSpaceNotFound = 404, kHostNotFound = 405, diff --git a/src/common/base/test/CMakeLists.txt b/src/common/base/test/CMakeLists.txt index 4963639cdf3..e644906d920 100644 --- a/src/common/base/test/CMakeLists.txt +++ b/src/common/base/test/CMakeLists.txt @@ -110,8 +110,10 @@ nebula_add_executable( $ $ $ + $ $ $ + $ $ $ $ diff --git a/src/common/datatypes/DataSetOps-inl.h b/src/common/datatypes/DataSetOps-inl.h index 2290df8bd75..89207de8497 100644 --- a/src/common/datatypes/DataSetOps-inl.h +++ b/src/common/datatypes/DataSetOps-inl.h @@ -13,6 +13,7 @@ #include "common/base/Base.h" #include "common/datatypes/CommonCpp2Ops.h" #include "common/datatypes/DataSet.h" +#include "common/memory/MemoryTracker.h" namespace apache { namespace thrift { @@ -47,7 +48,10 @@ inline constexpr protocol::TType Cpp2Ops::thriftType() { template uint32_t Cpp2Ops::write(Protocol* proto, nebula::DataSet const* obj) { + // we do not turn on memory tracker here, when the DataSet object is creating & inserting, it is + // in Processor::process(), where memory tracker is turned on. so we think that is enough. uint32_t xfer = 0; + xfer += proto->writeStructBegin("DataSet"); xfer += proto->writeFieldBegin("column_names", protocol::T_LIST, 1); @@ -62,11 +66,20 @@ uint32_t Cpp2Ops::write(Protocol* proto, nebula::DataSet const* xfer += proto->writeFieldStop(); xfer += proto->writeStructEnd(); + return xfer; } template void Cpp2Ops::read(Protocol* proto, nebula::DataSet* obj) { + // memory usage during decode a StorageResponse should be mostly occupied + // by DataSet (see interface/storage.thrift), turn on memory check here. + // + // MemoryTrackerVerified: + // throw std::bad_alloc has verified, can be captured in + // StorageClientBase::getResponse's onError + nebula::memory::MemoryCheckGuard guard; + apache::thrift::detail::ProtocolReaderStructReadState readState; readState.readStructBegin(proto); diff --git a/src/common/datatypes/ValueOps-inl.h b/src/common/datatypes/ValueOps-inl.h index d9ab04320f4..c58cb0a5261 100644 --- a/src/common/datatypes/ValueOps-inl.h +++ b/src/common/datatypes/ValueOps-inl.h @@ -103,6 +103,7 @@ template uint32_t Cpp2Ops::write(Protocol* proto, nebula::Value const* obj) { uint32_t xfer = 0; xfer += proto->writeStructBegin("Value"); + // MemoryTrackerVerified: throw bad_alloc verified switch (obj->type()) { case nebula::Value::Type::NULLVALUE: { diff --git a/src/common/datatypes/test/CMakeLists.txt b/src/common/datatypes/test/CMakeLists.txt index 0fdc5fde32c..f31731cf58b 100644 --- a/src/common/datatypes/test/CMakeLists.txt +++ b/src/common/datatypes/test/CMakeLists.txt @@ -43,6 +43,7 @@ nebula_add_test( $ $ $ + $ $ $ $ @@ -115,6 +116,7 @@ nebula_add_test( $ $ $ + $ $ $ LIBRARIES diff --git a/src/common/expression/test/CMakeLists.txt b/src/common/expression/test/CMakeLists.txt index b2884aecd67..d2c8b304859 100644 --- a/src/common/expression/test/CMakeLists.txt +++ b/src/common/expression/test/CMakeLists.txt @@ -128,6 +128,7 @@ nebula_add_executable( $ $ $ + $ $ $ $ @@ -153,6 +154,7 @@ nebula_add_executable( $ $ $ + $ $ $ $ diff --git a/src/common/geo/test/CMakeLists.txt b/src/common/geo/test/CMakeLists.txt index 932fffb14e6..5b37c79ffb9 100644 --- a/src/common/geo/test/CMakeLists.txt +++ b/src/common/geo/test/CMakeLists.txt @@ -30,6 +30,9 @@ nebula_add_test( $ $ $ + $ + $ + $ $ $ $ diff --git a/src/common/graph/tests/CMakeLists.txt b/src/common/graph/tests/CMakeLists.txt index cba41ef98ae..83465cf8f09 100644 --- a/src/common/graph/tests/CMakeLists.txt +++ b/src/common/graph/tests/CMakeLists.txt @@ -11,6 +11,9 @@ nebula_add_test( $ $ $ + $ + $ + $ $ $ $ diff --git a/src/common/id/test/CMakeLists.txt b/src/common/id/test/CMakeLists.txt index bcd640bd625..afd71499bdc 100644 --- a/src/common/id/test/CMakeLists.txt +++ b/src/common/id/test/CMakeLists.txt @@ -28,6 +28,7 @@ nebula_add_executable( $ $ $ + $ $ $ $ @@ -67,6 +68,7 @@ nebula_add_test( $ $ $ + $ $ $ $ diff --git a/src/common/memory/MemoryTracker.cpp b/src/common/memory/MemoryTracker.cpp index d7bc96432a4..548e4aacb38 100644 --- a/src/common/memory/MemoryTracker.cpp +++ b/src/common/memory/MemoryTracker.cpp @@ -38,6 +38,10 @@ void MemoryTracker::free(int64_t size) { MemoryStats::instance().free(size); } +bool MemoryTracker::isOn() { + return MemoryStats::instance().throwOnMemoryExceeded(); +} + void MemoryTracker::allocImpl(int64_t size, bool) { MemoryStats::instance().alloc(size); } diff --git a/src/common/memory/MemoryTracker.h b/src/common/memory/MemoryTracker.h index 1d2f344d56f..f42d5223ad9 100644 --- a/src/common/memory/MemoryTracker.h +++ b/src/common/memory/MemoryTracker.h @@ -138,6 +138,11 @@ class MemoryStats { threadMemoryStats_.throwOnMemoryExceeded = false; } + // return true if current thread's throwOnMemoryExceeded' + static bool throwOnMemoryExceeded() { + return threadMemoryStats_.throwOnMemoryExceeded; + } + private: inline ALWAYS_INLINE void allocGlobal(int64_t size) { int64_t willBe = size + used_.fetch_add(size, std::memory_order_relaxed); @@ -182,6 +187,9 @@ struct MemoryTracker { /// This function should be called after memory deallocation. static void free(int64_t size); + /// Test state of memory tracker, return true if memory tracker is turned on, otherwise false. + static bool isOn(); + private: static void allocImpl(int64_t size, bool throw_if_memory_exceeded); }; diff --git a/src/common/memory/MemoryUtils.cpp b/src/common/memory/MemoryUtils.cpp index aded2dd96cf..1d7170baffb 100644 --- a/src/common/memory/MemoryUtils.cpp +++ b/src/common/memory/MemoryUtils.cpp @@ -119,7 +119,8 @@ StatusOr MemoryUtils::hitsHighWatermark() { } // MemoryStats depends on jemalloc -#if ENABLE_JEMALLOC +#ifdef ENABLE_JEMALLOC +#ifndef ENABLE_ASAN // set MemoryStats limit (MemoryTracker track-able memory) int64_t trackable = total - FLAGS_memory_tracker_untracked_reserved_memory_mb * MiB; if (trackable > 0) { @@ -134,16 +135,9 @@ StatusOr MemoryUtils::hitsHighWatermark() { if (FLAGS_memory_purge_enabled) { int64_t now = time::WallClock::fastNowInSec(); if (now - kLastPurge_ > FLAGS_memory_purge_interval_seconds) { - // mallctl seems has issue with address_sanitizer, do purge only when address_sanitizer is off -#if defined(__clang__) -#if defined(__has_feature) -#if not __has_feature(address_sanitizer) - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); -#endif -#endif -#else // gcc + // jemalloc seems has issue with address_sanitizer, do purge only when address_sanitizer is + // off mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); -#endif kLastPurge_ = now; } } @@ -173,7 +167,7 @@ StatusOr MemoryUtils::hitsHighWatermark() { kLastPrintMemoryTrackerStats_ = now; } } - +#endif #endif auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio; diff --git a/src/common/memory/NewDelete.cpp b/src/common/memory/NewDelete.cpp index 4ff129cf331..5bc67076553 100644 --- a/src/common/memory/NewDelete.cpp +++ b/src/common/memory/NewDelete.cpp @@ -11,23 +11,16 @@ /// Two condition need check before MemoryTracker is on /// 1. jemalloc is used /// MemoryTracker need jemalloc API to get accurate size of alloc/free memory. -#if ENABLE_JEMALLOC /// 2. address_sanitizer is off /// sanitizer has already override the new/delete operator, /// only override new/delete operator only when address_sanitizer is off -#if defined(__clang__) -#if defined(__has_feature) -#if not __has_feature(address_sanitizer) +#ifdef ENABLE_JEMALLOC +#ifndef ENABLE_ASAN #define ENABLE_MEMORY_TRACKER #endif #endif -#else // gcc -#define ENABLE_MEMORY_TRACKER -#endif -#endif - -#if defined(ENABLE_MEMORY_TRACKER) +#ifdef ENABLE_MEMORY_TRACKER /// new void *operator new(std::size_t size) { nebula::memory::trackMemory(size); diff --git a/src/common/utils/test/CMakeLists.txt b/src/common/utils/test/CMakeLists.txt index afbdc3e9de4..63727036237 100644 --- a/src/common/utils/test/CMakeLists.txt +++ b/src/common/utils/test/CMakeLists.txt @@ -21,6 +21,9 @@ nebula_add_test( $ $ $ + $ + $ + $ $ $ LIBRARIES @@ -47,6 +50,9 @@ nebula_add_test( $ $ $ + $ + $ + $ $ $ LIBRARIES @@ -76,6 +82,9 @@ nebula_add_test( $ $ $ + $ + $ + $ $ $ LIBRARIES @@ -94,6 +103,8 @@ nebula_add_test( $ $ $ + $ + $ $ $ $ diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index e3d62265b8b..313d8cd7f59 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -603,6 +603,8 @@ Status Executor::open() { } Status Executor::close() { + // MemoryTrackerVerified + ProfilingStats stats; stats.totalDurationInUs = totalDuration_.elapsedInUSec(); stats.rows = numRows_; @@ -725,6 +727,7 @@ bool Executor::movable(const Variable *var) { } Status Executor::finish(Result &&result) { + // MemoryTrackerVerified if (!FLAGS_enable_lifetime_optimize || node()->outputVarPtr()->userCount.load(std::memory_order_relaxed) != 0) { numRows_ = result.size(); diff --git a/src/graph/executor/Executor.h b/src/graph/executor/Executor.h index 670eb4df302..ba9f90970cd 100644 --- a/src/graph/executor/Executor.h +++ b/src/graph/executor/Executor.h @@ -77,8 +77,8 @@ class Executor : private boost::noncopyable, private cpp::NonMovable { folly::Future error(Status status) const; static Status memoryExceededStatus() { - return Status::Error("Graph Error: GRAPH_MEMORY_EXCEEDED(%d)", - static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED)); + return Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED)); } protected: @@ -159,6 +159,7 @@ auto Executor::runMultiJobs(ScatterFunc &&scatter, GatherFunc &&gather, Iterator futures.emplace_back(folly::via( runner(), [begin, end, tmpIter = iter->copy(), f = std::move(scatter)]() mutable -> ScatterResult { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; // Since not all iterators are linear, so iterates to the begin pos size_t tmp = 0; @@ -173,17 +174,7 @@ auto Executor::runMultiJobs(ScatterFunc &&scatter, GatherFunc &&gather, Iterator } // Gather all results and do post works - return folly::collect(futures) - .via(runner()) - .thenValue(std::move(gather)) - .thenError(folly::tag_t{}, - [](const std::bad_alloc &) { - return folly::makeFuture(std::runtime_error( - "Memory Limit Exceeded, " + memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + return folly::collect(futures).via(runner()).thenValue(std::move(gather)); } } // namespace graph } // namespace nebula diff --git a/src/graph/executor/StorageAccessExecutor.h b/src/graph/executor/StorageAccessExecutor.h index 18ddbe83e40..a3881325f7c 100644 --- a/src/graph/executor/StorageAccessExecutor.h +++ b/src/graph/executor/StorageAccessExecutor.h @@ -131,9 +131,11 @@ class StorageAccessExecutor : public Executor { "Storage Error: Part {} raft buffer is full. Please retry later.", partId)); case nebula::cpp2::ErrorCode::E_RAFT_ATOMIC_OP_FAILED: return Status::Error("Storage Error: Atomic operation failed."); + // E_GRAPH_MEMORY_EXCEEDED may happen during rpc response deserialize. + case nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED: + return Status::GraphMemoryExceeded("(%d)", static_cast(code)); case nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED: - return Status::Error("Storage Error: STORAGE_MEMORY_EXCEEDED(%d)", - static_cast(code)); + return Status::StorageMemoryExceeded("(%d)", static_cast(code)); default: auto status = Status::Error("Storage Error: part: %d, error: %s(%d).", partId, diff --git a/src/graph/executor/algo/BFSShortestPathExecutor.cpp b/src/graph/executor/algo/BFSShortestPathExecutor.cpp index 7cb4fda3447..cf9faafa0d9 100644 --- a/src/graph/executor/algo/BFSShortestPathExecutor.cpp +++ b/src/graph/executor/algo/BFSShortestPathExecutor.cpp @@ -10,6 +10,8 @@ DECLARE_int32(num_operator_threads); namespace nebula { namespace graph { folly::Future BFSShortestPathExecutor::execute() { + // MemoryTrackerVerified + SCOPED_TIMER(&execTime_); pathNode_ = asNode(node()); terminateEarlyVar_ = pathNode_->terminateEarlyVar(); @@ -30,10 +32,12 @@ folly::Future BFSShortestPathExecutor::execute() { std::vector> futures; auto leftFuture = folly::via(runner(), [this]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return buildPath(false); }); auto rightFuture = folly::via(runner(), [this]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return buildPath(true); }); @@ -111,6 +115,7 @@ Status BFSShortestPathExecutor::buildPath(bool reverse) { currentEdges.emplace(std::move(dst), std::move(edge)); } } + // set nextVid const auto& nextVidVar = reverse ? pathNode_->rightVidVar() : pathNode_->leftVidVar(); ectx_->setResult(nextVidVar, ResultBuilder().value(std::move(nextStepVids)).build()); @@ -170,12 +175,6 @@ folly::Future BFSShortestPathExecutor::conjunctPath() { currentDs_.append(std::move(resp)); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/algo/BatchShortestPath.cpp b/src/graph/executor/algo/BatchShortestPath.cpp index 2400364b592..95ac030019d 100644 --- a/src/graph/executor/algo/BatchShortestPath.cpp +++ b/src/graph/executor/algo/BatchShortestPath.cpp @@ -15,6 +15,7 @@ namespace graph { folly::Future BatchShortestPath::execute(const HashSet& startVids, const HashSet& endVids, DataSet* result) { + // MemoryTrackerVerified size_t rowSize = init(startVids, endVids); std::vector> futures; futures.reserve(rowSize); @@ -22,26 +23,18 @@ folly::Future BatchShortestPath::execute(const HashSet& startVids, resultDs_[rowNum].colNames = pathNode_->colNames(); futures.emplace_back(shortestPath(rowNum, 1)); } - return folly::collect(futures) - .via(qctx_->rctx()->runner()) - .thenValue([this, result](auto&& resps) { - memory::MemoryCheckGuard guard; - for (auto& resp : resps) { - NG_RETURN_IF_ERROR(resp); - } - result->colNames = pathNode_->colNames(); - for (auto& ds : resultDs_) { - result->append(std::move(ds)); - } - return Status::OK(); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + return folly::collect(futures).via(runner()).thenValue([this, result](auto&& resps) { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + for (auto& resp : resps) { + NG_RETURN_IF_ERROR(resp); + } + result->colNames = pathNode_->colNames(); + for (auto& ds : resultDs_) { + result->append(std::move(ds)); + } + return Status::OK(); + }); } size_t BatchShortestPath::init(const HashSet& startVids, const HashSet& endVids) { @@ -106,8 +99,9 @@ folly::Future BatchShortestPath::shortestPath(size_t rowNum, size_t step futures.emplace_back(getNeighbors(rowNum, stepNum, false)); futures.emplace_back(getNeighbors(rowNum, stepNum, true)); return folly::collect(futures) - .via(qctx_->rctx()->runner()) + .via(runner()) .thenValue([this, rowNum, stepNum](auto&& resps) { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; for (auto& resp : resps) { if (!resp.ok()) { @@ -116,6 +110,10 @@ folly::Future BatchShortestPath::shortestPath(size_t rowNum, size_t step } return handleResponse(rowNum, stepNum); }) + // This thenError is necessary to catch bad_alloc, seems the returned future + // is related to two routines: getNeighbors, handleResponse, each of them launch some task in + // separate thread, if any one of routine throw bad_alloc, fail the query, will cause another + // to run on a maybe already released BatchShortestPath object .thenError(folly::tag_t{}, [](const std::bad_alloc&) { return folly::makeFuture(Executor::memoryExceededStatus()); @@ -151,18 +149,12 @@ folly::Future BatchShortestPath::getNeighbors(size_t rowNum, size_t step -1, nullptr, nullptr) - .via(qctx_->rctx()->runner()) + .via(runner()) .thenValue([this, rowNum, reverse, stepNum, getNbrTime](auto&& resp) { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; addStats(resp, stepNum, getNbrTime.elapsedInUSec(), reverse); return buildPath(rowNum, std::move(resp), reverse); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -281,14 +273,19 @@ Status BatchShortestPath::doBuildPath(size_t rowNum, GetNeighborsIter* iter, boo folly::Future BatchShortestPath::handleResponse(size_t rowNum, size_t stepNum) { return folly::makeFuture(Status::OK()) - .via(qctx_->rctx()->runner()) + .via(runner()) .thenValue([this, rowNum](auto&& status) { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; + // odd step UNUSED(status); return conjunctPath(rowNum, true); }) .thenValue([this, rowNum, stepNum](auto&& terminate) { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + // even Step if (terminate || stepNum * 2 > maxStep_) { return folly::makeFuture(true); @@ -296,6 +293,9 @@ folly::Future BatchShortestPath::handleResponse(size_t rowNum, size_t st return conjunctPath(rowNum, false); }) .thenValue([this, rowNum, stepNum](auto&& result) { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + if (result || stepNum * 2 >= maxStep_) { return folly::makeFuture(Status::OK()); } @@ -319,13 +319,6 @@ folly::Future BatchShortestPath::handleResponse(size_t rowNum, size_t st leftPathMap.clear(); rightPathMap.clear(); return shortestPath(rowNum, stepNum + 1); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -379,64 +372,61 @@ folly::Future BatchShortestPath::conjunctPath(size_t rowNum, bool oddStep) } auto future = getMeetVids(rowNum, oddStep, meetVids); - return future.via(qctx_->rctx()->runner()) - .thenValue([this, rowNum, oddStep](auto&& vertices) { - memory::MemoryCheckGuard guard; - if (vertices.empty()) { - return false; - } - robin_hood::unordered_flat_map> verticesMap; - for (auto& vertex : vertices) { - verticesMap[vertex.getVertex().vid] = std::move(vertex); + return future.via(runner()).thenValue([this, rowNum, oddStep](auto&& vertices) { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + + if (vertices.empty()) { + return false; + } + robin_hood::unordered_flat_map> verticesMap; + for (auto& vertex : vertices) { + verticesMap[vertex.getVertex().vid] = std::move(vertex); + } + auto& terminationMap = terminationMaps_[rowNum]; + auto& leftPathMaps = currentLeftPathMaps_[rowNum]; + auto& rightPathMaps = oddStep ? preRightPathMaps_[rowNum] : currentRightPathMaps_[rowNum]; + for (const auto& leftPathMap : leftPathMaps) { + auto findCommonVid = rightPathMaps.find(leftPathMap.first); + if (findCommonVid == rightPathMaps.end()) { + continue; + } + auto findCommonVertex = verticesMap.find(findCommonVid->first); + if (findCommonVertex == verticesMap.end()) { + continue; + } + auto& rightPaths = findCommonVid->second; + for (const auto& srcPaths : leftPathMap.second) { + auto range = terminationMap.equal_range(srcPaths.first); + if (range.first == range.second) { + continue; } - auto& terminationMap = terminationMaps_[rowNum]; - auto& leftPathMaps = currentLeftPathMaps_[rowNum]; - auto& rightPathMaps = oddStep ? preRightPathMaps_[rowNum] : currentRightPathMaps_[rowNum]; - for (const auto& leftPathMap : leftPathMaps) { - auto findCommonVid = rightPathMaps.find(leftPathMap.first); - if (findCommonVid == rightPathMaps.end()) { - continue; - } - auto findCommonVertex = verticesMap.find(findCommonVid->first); - if (findCommonVertex == verticesMap.end()) { - continue; - } - auto& rightPaths = findCommonVid->second; - for (const auto& srcPaths : leftPathMap.second) { - auto range = terminationMap.equal_range(srcPaths.first); - if (range.first == range.second) { - continue; - } - for (const auto& dstPaths : rightPaths) { - for (auto found = range.first; found != range.second; ++found) { - if (found->second.first == dstPaths.first) { - if (singleShortest_ && !found->second.second) { - break; - } - doConjunctPath( - srcPaths.second, dstPaths.second, findCommonVertex->second, rowNum); - found->second.second = false; - } + for (const auto& dstPaths : rightPaths) { + for (auto found = range.first; found != range.second; ++found) { + if (found->second.first == dstPaths.first) { + if (singleShortest_ && !found->second.second) { + break; } + doConjunctPath(srcPaths.second, dstPaths.second, findCommonVertex->second, rowNum); + found->second.second = false; } } } - // update terminationMap - for (auto iter = terminationMap.begin(); iter != terminationMap.end();) { - if (!iter->second.second) { - iter = terminationMap.erase(iter); - } else { - ++iter; - } - } - if (terminationMap.empty()) { - return true; - } - return false; - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + } + } + // update terminationMap + for (auto iter = terminationMap.begin(); iter != terminationMap.end();) { + if (!iter->second.second) { + iter = terminationMap.erase(iter); + } else { + ++iter; + } + } + if (terminationMap.empty()) { + return true; + } + return false; + }); } void BatchShortestPath::doConjunctPath(const std::vector& leftPaths, diff --git a/src/graph/executor/algo/MultiShortestPathExecutor.cpp b/src/graph/executor/algo/MultiShortestPathExecutor.cpp index f77dc4d9d59..d9f0fddff15 100644 --- a/src/graph/executor/algo/MultiShortestPathExecutor.cpp +++ b/src/graph/executor/algo/MultiShortestPathExecutor.cpp @@ -46,6 +46,7 @@ folly::Future MultiShortestPathExecutor::execute() { return conjunctPath(false); }) .thenValue([this](auto&& resp) { + memory::MemoryCheckGuard guard; UNUSED(resp); preRightPaths_ = rightPaths_; // update history @@ -311,30 +312,25 @@ folly::Future MultiShortestPathExecutor::conjunctPath(bool oddStep) { futures.emplace_back(std::move(future)); } - return folly::collect(futures) - .via(runner()) - .thenValue([this](auto&& resps) { - memory::MemoryCheckGuard guard; - for (auto& resp : resps) { - currentDs_.append(std::move(resp)); - } + return folly::collect(futures).via(runner()).thenValue([this](auto&& resps) { + memory::MemoryCheckGuard guard; + for (auto& resp : resps) { + currentDs_.append(std::move(resp)); + } - for (auto iter = terminationMap_.begin(); iter != terminationMap_.end();) { - if (!iter->second.second) { - iter = terminationMap_.erase(iter); - } else { - ++iter; - } - } - if (terminationMap_.empty()) { - ectx_->setValue(terminationVar_, true); - return true; - } - return false; - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + for (auto iter = terminationMap_.begin(); iter != terminationMap_.end();) { + if (!iter->second.second) { + iter = terminationMap_.erase(iter); + } else { + ++iter; + } + } + if (terminationMap_.empty()) { + ectx_->setValue(terminationVar_, true); + return true; + } + return false; + }); } void MultiShortestPathExecutor::setNextStepVid(const Interims& paths, const string& var) { diff --git a/src/graph/executor/algo/ProduceAllPathsExecutor.cpp b/src/graph/executor/algo/ProduceAllPathsExecutor.cpp index 44e771ab7cd..f2b709a000f 100644 --- a/src/graph/executor/algo/ProduceAllPathsExecutor.cpp +++ b/src/graph/executor/algo/ProduceAllPathsExecutor.cpp @@ -28,10 +28,12 @@ folly::Future ProduceAllPathsExecutor::execute() { } std::vector> futures; auto leftFuture = folly::via(runner(), [this]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return buildPath(false); }); auto rightFuture = folly::via(runner(), [this]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return buildPath(true); }); @@ -156,12 +158,14 @@ folly::Future ProduceAllPathsExecutor::conjunctPath() { auto endIter = leftIter; endIter++; auto oddStepFuture = folly::via(runner(), [this, startIter, endIter]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return doConjunct(startIter, endIter, true); }); futures.emplace_back(std::move(oddStepFuture)); if (step_ * 2 <= pathNode_->steps()) { auto evenStepFuture = folly::via(runner(), [this, startIter, endIter]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return doConjunct(startIter, endIter, false); }); @@ -175,12 +179,14 @@ folly::Future ProduceAllPathsExecutor::conjunctPath() { if (i != 0) { auto endIter = leftPaths_.end(); auto oddStepFuture = folly::via(runner(), [this, startIter, endIter]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return doConjunct(startIter, endIter, true); }); futures.emplace_back(std::move(oddStepFuture)); if (step_ * 2 <= pathNode_->steps()) { auto evenStepFuture = folly::via(runner(), [this, startIter, endIter]() { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; return doConjunct(startIter, endIter, false); }); @@ -200,12 +206,6 @@ folly::Future ProduceAllPathsExecutor::conjunctPath() { leftPaths_.clear(); rightPaths_.clear(); return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/algo/ShortestPathBase.cpp b/src/graph/executor/algo/ShortestPathBase.cpp index 9bc82ed0ed9..e809f27f72b 100644 --- a/src/graph/executor/algo/ShortestPathBase.cpp +++ b/src/graph/executor/algo/ShortestPathBase.cpp @@ -197,5 +197,12 @@ void ShortestPathBase::addStats(PropRpcResponse& resp, int64_t timeInUSec) const statsLock_.unlock(); } +folly::Executor* ShortestPathBase::runner() const { + if (!qctx_ || !qctx_->rctx() || !qctx_->rctx()->runner()) { + return &folly::InlineExecutor::instance(); + } + return qctx_->rctx()->runner(); +} + } // namespace graph } // namespace nebula diff --git a/src/graph/executor/algo/ShortestPathBase.h b/src/graph/executor/algo/ShortestPathBase.h index 9fd03f42305..2a049246a74 100644 --- a/src/graph/executor/algo/ShortestPathBase.h +++ b/src/graph/executor/algo/ShortestPathBase.h @@ -51,6 +51,8 @@ class ShortestPathBase { void addStats(PropRpcResponse& resp, int64_t timeInUSec) const; + folly::Executor* runner() const; + template StatusOr handleCompleteness(const storage::StorageRpcResponse& rpcResp, bool isPartialSuccessAccepted) const { diff --git a/src/graph/executor/algo/ShortestPathExecutor.cpp b/src/graph/executor/algo/ShortestPathExecutor.cpp index d8269ae074f..98381a9e72a 100644 --- a/src/graph/executor/algo/ShortestPathExecutor.cpp +++ b/src/graph/executor/algo/ShortestPathExecutor.cpp @@ -3,6 +3,7 @@ // This source code is licensed under Apache 2.0 License. #include "graph/executor/algo/ShortestPathExecutor.h" +#include "common/memory/MemoryTracker.h" #include "graph/executor/algo/BatchShortestPath.h" #include "graph/executor/algo/SingleShortestPath.h" #include "graph/service/GraphFlags.h" @@ -15,6 +16,7 @@ DEFINE_uint32(num_path_thread, 0, "number of concurrent threads when do shortest namespace nebula { namespace graph { folly::Future ShortestPathExecutor::execute() { + // MemoryTrackerVerified SCOPED_TIMER(&execTime_); if (FLAGS_num_path_thread == 0) { FLAGS_num_path_thread = get_nprocs_conf(); diff --git a/src/graph/executor/algo/SingleShortestPath.cpp b/src/graph/executor/algo/SingleShortestPath.cpp index 69d3dba483a..fd509d765f6 100644 --- a/src/graph/executor/algo/SingleShortestPath.cpp +++ b/src/graph/executor/algo/SingleShortestPath.cpp @@ -21,26 +21,17 @@ folly::Future SingleShortestPath::execute(const HashSet& startVids, resultDs_[rowNum].colNames = pathNode_->colNames(); futures.emplace_back(shortestPath(rowNum, 1)); } - return folly::collect(futures) - .via(qctx_->rctx()->runner()) - .thenValue([this, result](auto&& resps) { - memory::MemoryCheckGuard guard; - for (auto& resp : resps) { - NG_RETURN_IF_ERROR(resp); - } - result->colNames = pathNode_->colNames(); - for (auto& ds : resultDs_) { - result->append(std::move(ds)); - } - return Status::OK(); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + return folly::collect(futures).via(runner()).thenValue([this, result](auto&& resps) { + memory::MemoryCheckGuard guard; + for (auto& resp : resps) { + NG_RETURN_IF_ERROR(resp); + } + result->colNames = pathNode_->colNames(); + for (auto& ds : resultDs_) { + result->append(std::move(ds)); + } + return Status::OK(); + }); } void SingleShortestPath::init(const HashSet& startVids, const HashSet& endVids, size_t rowSize) { @@ -75,7 +66,7 @@ folly::Future SingleShortestPath::shortestPath(size_t rowNum, size_t ste futures.emplace_back(getNeighbors(rowNum, stepNum, false)); futures.emplace_back(getNeighbors(rowNum, stepNum, true)); return folly::collect(futures) - .via(qctx_->rctx()->runner()) + .via(runner()) .thenValue([this, rowNum, stepNum](auto&& resps) { memory::MemoryCheckGuard guard; for (auto& resp : resps) { @@ -122,18 +113,11 @@ folly::Future SingleShortestPath::getNeighbors(size_t rowNum, -1, nullptr, nullptr) - .via(qctx_->rctx()->runner()) + .via(runner()) .thenValue([this, rowNum, stepNum, getNbrTime, reverse](auto&& resp) { memory::MemoryCheckGuard guard; addStats(resp, stepNum, getNbrTime.elapsedInUSec(), reverse); return buildPath(rowNum, std::move(resp), reverse); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -198,13 +182,16 @@ Status SingleShortestPath::doBuildPath(size_t rowNum, GetNeighborsIter* iter, bo folly::Future SingleShortestPath::handleResponse(size_t rowNum, size_t stepNum) { return folly::makeFuture(Status::OK()) - .via(qctx_->rctx()->runner()) + .via(runner()) .thenValue([this, rowNum, stepNum](auto&& status) { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; + UNUSED(status); return conjunctPath(rowNum, stepNum); }) .thenValue([this, rowNum, stepNum](auto&& result) { + memory::MemoryCheckGuard guard; if (result || stepNum * 2 >= maxStep_) { return folly::makeFuture(Status::OK()); } @@ -214,13 +201,6 @@ folly::Future SingleShortestPath::handleResponse(size_t rowNum, size_t s return folly::makeFuture(Status::OK()); } return shortestPath(rowNum, stepNum + 1); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -281,38 +261,36 @@ void SingleShortestPath::buildOddPath(size_t rowNum, const std::vector& m folly::Future SingleShortestPath::buildEvenPath(size_t rowNum, const std::vector& meetVids) { auto future = getMeetVidsProps(meetVids); - return future.via(qctx_->rctx()->runner()) - .thenValue([this, rowNum](auto&& vertices) { - memory::MemoryCheckGuard guard; - if (vertices.empty()) { - return false; - } - for (auto& meetVertex : vertices) { - if (!meetVertex.isVertex()) { - continue; - } - auto meetVid = meetVertex.getVertex().vid; - auto leftPaths = createLeftPath(rowNum, meetVid); - auto rightPaths = createRightPath(rowNum, meetVid, false); - for (auto& leftPath : leftPaths) { - for (auto& rightPath : rightPaths) { - Row path = leftPath; - auto& steps = path.values.back().mutableList().values; - steps.emplace_back(meetVertex); - steps.insert(steps.end(), rightPath.values.begin(), rightPath.values.end() - 1); - path.emplace_back(rightPath.values.back()); - resultDs_[rowNum].rows.emplace_back(std::move(path)); - if (singleShortest_) { - return true; - } - } + return future.via(runner()).thenValue([this, rowNum](auto&& vertices) { + // MemoryTrackerVerified + memory::MemoryCheckGuard guard; + + if (vertices.empty()) { + return false; + } + for (auto& meetVertex : vertices) { + if (!meetVertex.isVertex()) { + continue; + } + auto meetVid = meetVertex.getVertex().vid; + auto leftPaths = createLeftPath(rowNum, meetVid); + auto rightPaths = createRightPath(rowNum, meetVid, false); + for (auto& leftPath : leftPaths) { + for (auto& rightPath : rightPaths) { + Row path = leftPath; + auto& steps = path.values.back().mutableList().values; + steps.emplace_back(meetVertex); + steps.insert(steps.end(), rightPath.values.begin(), rightPath.values.end() - 1); + path.emplace_back(rightPath.values.back()); + resultDs_[rowNum].rows.emplace_back(std::move(path)); + if (singleShortest_) { + return true; } } - return true; - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + } + } + return true; + }); } std::vector SingleShortestPath::createLeftPath(size_t rowNum, const Value& meetVid) { diff --git a/src/graph/executor/algo/SubgraphExecutor.cpp b/src/graph/executor/algo/SubgraphExecutor.cpp index e51f4853c0c..cb3e9ca7634 100644 --- a/src/graph/executor/algo/SubgraphExecutor.cpp +++ b/src/graph/executor/algo/SubgraphExecutor.cpp @@ -53,6 +53,7 @@ folly::Future SubgraphExecutor::getNeighbors() { currentStep_ == 1 ? nullptr : subgraph_->tagFilter()) .via(runner()) .thenValue([this, getNbrTime](RpcResponse&& resp) mutable { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; otherStats_.emplace("total_rpc_time", folly::sformat("{}(us)", getNbrTime.elapsedInUSec())); auto& hostLatency = resp.hostLatency(); @@ -67,12 +68,6 @@ folly::Future SubgraphExecutor::getNeighbors() { } vids_.clear(); return handleResponse(std::move(resp)); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/logic/ArgumentExecutor.cpp b/src/graph/executor/logic/ArgumentExecutor.cpp index eab46383f17..1e74f3ba165 100644 --- a/src/graph/executor/logic/ArgumentExecutor.cpp +++ b/src/graph/executor/logic/ArgumentExecutor.cpp @@ -13,6 +13,7 @@ ArgumentExecutor::ArgumentExecutor(const PlanNode *node, QueryContext *qctx) : Executor("ArgumentExecutor", node, qctx) {} folly::Future ArgumentExecutor::execute() { + // MemoryTrackerVerified auto *argNode = asNode(node()); auto &alias = argNode->getAlias(); auto iter = ectx_->getResult(argNode->inputVar()).iter(); diff --git a/src/graph/executor/maintain/EdgeExecutor.cpp b/src/graph/executor/maintain/EdgeExecutor.cpp index e198624af6a..1c8a3151f44 100644 --- a/src/graph/executor/maintain/EdgeExecutor.cpp +++ b/src/graph/executor/maintain/EdgeExecutor.cpp @@ -21,18 +21,13 @@ folly::Future CreateEdgeExecutor::execute() { .via(runner()) .thenValue([ceNode, spaceId](StatusOr resp) { memory::MemoryCheckGuard guard; + // throw in MemoryCheckGuard verified if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Create edge `" << ceNode->getName() << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -47,6 +42,7 @@ folly::Future DescEdgeExecutor::execute() { .via(runner()) .thenValue([this, deNode, spaceId](StatusOr resp) { memory::MemoryCheckGuard guard; + // MemoryTrackerVerified if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Desc edge `" << deNode->getName() << "' failed: " << resp.status(); @@ -62,12 +58,6 @@ folly::Future DescEdgeExecutor::execute() { .value(Value(std::move(ret).value())) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -82,18 +72,13 @@ folly::Future DropEdgeExecutor::execute() { .via(runner()) .thenValue([deNode, spaceId](StatusOr resp) { memory::MemoryCheckGuard guard; + // MemoryTrackerVerified if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Drop edge `" << deNode->getName() << "' failed: " << resp.status(); return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -128,12 +113,6 @@ folly::Future ShowEdgesExecutor::execute() { .value(Value(std::move(dataSet))) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -161,12 +140,6 @@ folly::Future ShowCreateEdgeExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -187,12 +160,6 @@ folly::Future AlterEdgeExecutor::execute() { return resp.status(); } return finish(ResultBuilder().value(Value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } } // namespace graph diff --git a/src/graph/executor/maintain/EdgeIndexExecutor.cpp b/src/graph/executor/maintain/EdgeIndexExecutor.cpp index 38d6a9f3087..00f1f2e9d20 100644 --- a/src/graph/executor/maintain/EdgeIndexExecutor.cpp +++ b/src/graph/executor/maintain/EdgeIndexExecutor.cpp @@ -27,6 +27,7 @@ folly::Future CreateEdgeIndexExecutor::execute() { .via(runner()) .thenValue([ceiNode, spaceId](StatusOr resp) { memory::MemoryCheckGuard guard; + // MemoryTrackerVerified if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Create index `" << ceiNode->getIndexName() << "' at edge: `" << ceiNode->getSchemaName() @@ -34,12 +35,6 @@ folly::Future CreateEdgeIndexExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -60,12 +55,6 @@ folly::Future DropEdgeIndexExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -94,12 +83,6 @@ folly::Future DescEdgeIndexExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -127,12 +110,6 @@ folly::Future ShowCreateEdgeIndexExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -190,12 +167,6 @@ folly::Future ShowEdgeIndexesExecutor::execute() { .value(Value(std::move(dataSet))) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -229,12 +200,6 @@ folly::Future ShowEdgeIndexStatusExecutor::execute() { .value(Value(std::move(dataSet))) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/maintain/FTIndexExecutor.cpp b/src/graph/executor/maintain/FTIndexExecutor.cpp index 06d08f91d07..2d174fcc025 100644 --- a/src/graph/executor/maintain/FTIndexExecutor.cpp +++ b/src/graph/executor/maintain/FTIndexExecutor.cpp @@ -26,12 +26,6 @@ folly::Future CreateFTIndexExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -59,67 +53,51 @@ folly::Future DropFTIndexExecutor::execute() { LOG(WARNING) << "Drop fulltext index '" << inode->getName() << "' failed: " << ftRet; } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } folly::Future ShowFTIndexesExecutor::execute() { SCOPED_TIMER(&execTime_); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listFTIndexes() - .via(runner()) - .thenValue( - [this, spaceId](StatusOr> resp) { - memory::MemoryCheckGuard guard; - if (!resp.ok()) { - LOG(WARNING) << "SpaceId: " << spaceId << ", Show fulltext indexes failed" - << resp.status(); - return resp.status(); - } + return qctx()->getMetaClient()->listFTIndexes().via(runner()).thenValue( + [this, spaceId](StatusOr> resp) { + memory::MemoryCheckGuard guard; + if (!resp.ok()) { + LOG(WARNING) << "SpaceId: " << spaceId << ", Show fulltext indexes failed" + << resp.status(); + return resp.status(); + } - auto indexes = std::move(resp).value(); - DataSet dataSet; - dataSet.colNames = {"Name", "Schema Type", "Schema Name", "Fields"}; - for (auto &index : indexes) { - if (index.second.get_space_id() != spaceId) { - continue; - } - auto shmId = index.second.get_depend_schema(); - auto isEdge = shmId.getType() == nebula::cpp2::SchemaID::Type::edge_type; - auto shmNameRet = - isEdge ? this->qctx_->schemaMng()->toEdgeName(spaceId, shmId.get_edge_type()) - : this->qctx_->schemaMng()->toTagName(spaceId, shmId.get_tag_id()); - if (!shmNameRet.ok()) { - LOG(WARNING) << "SpaceId: " << spaceId - << ", Get schema name failed: " << shmNameRet.status(); - return shmNameRet.status(); - } - std::string fields; - folly::join(", ", index.second.get_fields(), fields); - Row row; - row.values.emplace_back(index.first); - row.values.emplace_back(isEdge ? "Edge" : "Tag"); - row.values.emplace_back(std::move(shmNameRet).value()); - row.values.emplace_back(std::move(fields)); - dataSet.rows.emplace_back(std::move(row)); - } - return finish(ResultBuilder() - .value(Value(std::move(dataSet))) - .iter(Iterator::Kind::kDefault) - .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); + auto indexes = std::move(resp).value(); + DataSet dataSet; + dataSet.colNames = {"Name", "Schema Type", "Schema Name", "Fields"}; + for (auto &index : indexes) { + if (index.second.get_space_id() != spaceId) { + continue; + } + auto shmId = index.second.get_depend_schema(); + auto isEdge = shmId.getType() == nebula::cpp2::SchemaID::Type::edge_type; + auto shmNameRet = + isEdge ? this->qctx_->schemaMng()->toEdgeName(spaceId, shmId.get_edge_type()) + : this->qctx_->schemaMng()->toTagName(spaceId, shmId.get_tag_id()); + if (!shmNameRet.ok()) { + LOG(WARNING) << "SpaceId: " << spaceId + << ", Get schema name failed: " << shmNameRet.status(); + return shmNameRet.status(); + } + std::string fields; + folly::join(", ", index.second.get_fields(), fields); + Row row; + row.values.emplace_back(index.first); + row.values.emplace_back(isEdge ? "Edge" : "Tag"); + row.values.emplace_back(std::move(shmNameRet).value()); + row.values.emplace_back(std::move(fields)); + dataSet.rows.emplace_back(std::move(row)); + } + return finish(ResultBuilder() + .value(Value(std::move(dataSet))) + .iter(Iterator::Kind::kDefault) + .build()); }); } diff --git a/src/graph/executor/maintain/TagExecutor.cpp b/src/graph/executor/maintain/TagExecutor.cpp index 5beb4c0b1b6..5dd58b2e6fa 100644 --- a/src/graph/executor/maintain/TagExecutor.cpp +++ b/src/graph/executor/maintain/TagExecutor.cpp @@ -27,12 +27,6 @@ folly::Future CreateTagExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -60,12 +54,6 @@ folly::Future DescTagExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -86,12 +74,6 @@ folly::Future DropTagExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -99,11 +81,8 @@ folly::Future ShowTagsExecutor::execute() { SCOPED_TIMER(&execTime_); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listTagSchemas(spaceId) - .via(runner()) - .thenValue([this, spaceId](StatusOr> resp) { + return qctx()->getMetaClient()->listTagSchemas(spaceId).via(runner()).thenValue( + [this, spaceId](StatusOr> resp) { memory::MemoryCheckGuard guard; if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Show tags failed: " << resp.status(); @@ -126,12 +105,6 @@ folly::Future ShowTagsExecutor::execute() { .value(Value(std::move(dataSet))) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -159,12 +132,6 @@ folly::Future ShowCreateTagExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -185,12 +152,6 @@ folly::Future AlterTagExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } } // namespace graph diff --git a/src/graph/executor/maintain/TagIndexExecutor.cpp b/src/graph/executor/maintain/TagIndexExecutor.cpp index ad14dcd3788..41909588cb3 100644 --- a/src/graph/executor/maintain/TagIndexExecutor.cpp +++ b/src/graph/executor/maintain/TagIndexExecutor.cpp @@ -34,12 +34,6 @@ folly::Future CreateTagIndexExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -60,12 +54,6 @@ folly::Future DropTagIndexExecutor::execute() { return resp.status(); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -94,12 +82,6 @@ folly::Future DescTagIndexExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -127,12 +109,6 @@ folly::Future ShowCreateTagIndexExecutor::execute() { } return finish( ResultBuilder().value(std::move(ret).value()).iter(Iterator::Kind::kDefault).build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -141,11 +117,8 @@ folly::Future ShowTagIndexesExecutor::execute() { auto *iNode = asNode(node()); const auto &bySchema = iNode->name(); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listTagIndexes(spaceId) - .via(runner()) - .thenValue([this, spaceId, bySchema](StatusOr> resp) { + return qctx()->getMetaClient()->listTagIndexes(spaceId).via(runner()).thenValue( + [this, spaceId, bySchema](StatusOr> resp) { memory::MemoryCheckGuard guard; if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Show tag indexes failed" << resp.status(); @@ -191,12 +164,6 @@ folly::Future ShowTagIndexesExecutor::execute() { .value(Value(std::move(dataSet))) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -204,11 +171,8 @@ folly::Future ShowTagIndexStatusExecutor::execute() { SCOPED_TIMER(&execTime_); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listTagIndexStatus(spaceId) - .via(runner()) - .thenValue([this, spaceId](StatusOr> resp) { + return qctx()->getMetaClient()->listTagIndexStatus(spaceId).via(runner()).thenValue( + [this, spaceId](StatusOr> resp) { memory::MemoryCheckGuard guard; if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId @@ -230,12 +194,6 @@ folly::Future ShowTagIndexStatusExecutor::execute() { .value(Value(std::move(dataSet))) .iter(Iterator::Kind::kDefault) .build()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/mutate/DeleteExecutor.cpp b/src/graph/executor/mutate/DeleteExecutor.cpp index 71fee56ec86..15ef49a36b8 100644 --- a/src/graph/executor/mutate/DeleteExecutor.cpp +++ b/src/graph/executor/mutate/DeleteExecutor.cpp @@ -73,12 +73,6 @@ folly::Future DeleteVerticesExecutor::deleteVertices() { SCOPED_TIMER(&execTime_); NG_RETURN_IF_ERROR(handleCompleteness(resp, false)); return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -135,12 +129,6 @@ folly::Future DeleteTagsExecutor::deleteTags() { SCOPED_TIMER(&execTime_); NG_RETURN_IF_ERROR(handleCompleteness(resp, false)); return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -228,12 +216,6 @@ folly::Future DeleteEdgesExecutor::deleteEdges() { SCOPED_TIMER(&execTime_); NG_RETURN_IF_ERROR(handleCompleteness(resp, false)); return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } } // namespace graph diff --git a/src/graph/executor/mutate/InsertExecutor.cpp b/src/graph/executor/mutate/InsertExecutor.cpp index f41e867b39e..7fe31435d65 100644 --- a/src/graph/executor/mutate/InsertExecutor.cpp +++ b/src/graph/executor/mutate/InsertExecutor.cpp @@ -40,12 +40,6 @@ folly::Future InsertVerticesExecutor::insertVertices() { SCOPED_TIMER(&execTime_); NG_RETURN_IF_ERROR(handleCompleteness(resp, false)); return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -77,12 +71,6 @@ folly::Future InsertEdgesExecutor::insertEdges() { SCOPED_TIMER(&execTime_); NG_RETURN_IF_ERROR(handleCompleteness(resp, false)); return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } } // namespace graph diff --git a/src/graph/executor/mutate/UpdateExecutor.cpp b/src/graph/executor/mutate/UpdateExecutor.cpp index 89d6519cefe..3b820ac471a 100644 --- a/src/graph/executor/mutate/UpdateExecutor.cpp +++ b/src/graph/executor/mutate/UpdateExecutor.cpp @@ -82,12 +82,6 @@ folly::Future UpdateVertexExecutor::execute() { .build()); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -140,12 +134,6 @@ folly::Future UpdateEdgeExecutor::execute() { .build()); } return Status::OK(); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } } // namespace graph diff --git a/src/graph/executor/query/AggregateExecutor.cpp b/src/graph/executor/query/AggregateExecutor.cpp index 5c9fb046603..d9f8b5b5e30 100644 --- a/src/graph/executor/query/AggregateExecutor.cpp +++ b/src/graph/executor/query/AggregateExecutor.cpp @@ -11,6 +11,7 @@ namespace graph { folly::Future AggregateExecutor::execute() { SCOPED_TIMER(&execTime_); + // MemoryTrackerVerified auto* agg = asNode(node()); auto groupKeys = agg->groupKeys(); auto groupItems = agg->groupItems(); diff --git a/src/graph/executor/query/AppendVerticesExecutor.cpp b/src/graph/executor/query/AppendVerticesExecutor.cpp index 02bcd7bf81c..ae420b28ad4 100644 --- a/src/graph/executor/query/AppendVerticesExecutor.cpp +++ b/src/graph/executor/query/AppendVerticesExecutor.cpp @@ -61,6 +61,7 @@ folly::Future AppendVerticesExecutor::appendVertices() { otherStats_.emplace("total_rpc", folly::sformat("{}(us)", getPropsTime.elapsedInUSec())); }) .thenValue([this](StorageRpcResponse &&rpcResp) { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; SCOPED_TIMER(&execTime_); addStats(rpcResp, otherStats_); @@ -69,12 +70,6 @@ folly::Future AppendVerticesExecutor::appendVertices() { } else { return handleRespMultiJobs(std::move(rpcResp)); } - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/DedupExecutor.cpp b/src/graph/executor/query/DedupExecutor.cpp index 8e106f3537c..f569545fbe4 100644 --- a/src/graph/executor/query/DedupExecutor.cpp +++ b/src/graph/executor/query/DedupExecutor.cpp @@ -10,7 +10,9 @@ namespace nebula { namespace graph { folly::Future DedupExecutor::execute() { + // MemoryTrackerVerified SCOPED_TIMER(&execTime_); + auto* dedup = asNode(node()); DCHECK(!dedup->inputVar().empty()); Result result = ectx_->getResult(dedup->inputVar()); diff --git a/src/graph/executor/query/FilterExecutor.cpp b/src/graph/executor/query/FilterExecutor.cpp index 4f2e7334326..1b7a5f39fad 100644 --- a/src/graph/executor/query/FilterExecutor.cpp +++ b/src/graph/executor/query/FilterExecutor.cpp @@ -29,11 +29,14 @@ folly::Future FilterExecutor::execute() { ds.rows.reserve(iter->size()); auto scatter = [this]( size_t begin, size_t end, Iterator *tmpIter) mutable -> StatusOr { + // MemoryTrackerVerified + DCHECK(memory::MemoryTracker::isOn()) << "MemoryTracker is off"; return handleJob(begin, end, tmpIter); }; auto gather = [this, result = std::move(ds), kind = iter->kind()](auto &&results) mutable -> Status { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; for (auto &r : results) { if (!r.ok()) { diff --git a/src/graph/executor/query/GetDstBySrcExecutor.cpp b/src/graph/executor/query/GetDstBySrcExecutor.cpp index e694d43971f..5635d8106ef 100644 --- a/src/graph/executor/query/GetDstBySrcExecutor.cpp +++ b/src/graph/executor/query/GetDstBySrcExecutor.cpp @@ -61,12 +61,6 @@ folly::Future GetDstBySrcExecutor::execute() { otherStats_.emplace(folly::sformat("resp[{}]", i), folly::toPrettyJson(info)); } return handleResponse(resp, this->gd_->colNames()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/GetEdgesExecutor.cpp b/src/graph/executor/query/GetEdgesExecutor.cpp index 98306c39d51..eade60d9e2b 100644 --- a/src/graph/executor/query/GetEdgesExecutor.cpp +++ b/src/graph/executor/query/GetEdgesExecutor.cpp @@ -105,12 +105,6 @@ folly::Future GetEdgesExecutor::getEdges() { SCOPED_TIMER(&execTime_); addStats(rpcResp, otherStats_); return handleResp(std::move(rpcResp), ge->colNames()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp index 0b69967c41e..ae50cc283e8 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.cpp +++ b/src/graph/executor/query/GetNeighborsExecutor.cpp @@ -75,12 +75,6 @@ folly::Future GetNeighborsExecutor::execute() { otherStats_.emplace(folly::sformat("resp[{}]", i), folly::toPrettyJson(info)); } return handleResponse(resp); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/GetVerticesExecutor.cpp b/src/graph/executor/query/GetVerticesExecutor.cpp index 3906d3c564e..0d250044d9a 100644 --- a/src/graph/executor/query/GetVerticesExecutor.cpp +++ b/src/graph/executor/query/GetVerticesExecutor.cpp @@ -55,12 +55,6 @@ folly::Future GetVerticesExecutor::getVertices() { SCOPED_TIMER(&execTime_); addStats(rpcResp, otherStats_); return handleResp(std::move(rpcResp), gv->colNames()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp index e2bd6a65d9f..8bbfa8addd5 100644 --- a/src/graph/executor/query/IndexScanExecutor.cpp +++ b/src/graph/executor/query/IndexScanExecutor.cpp @@ -42,15 +42,10 @@ folly::Future IndexScanExecutor::indexScan() { lookup->limit(qctx_)) .via(runner()) .thenValue([this](StorageRpcResponse &&rpcResp) { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; addStats(rpcResp, otherStats_); return handleResp(std::move(rpcResp)); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/ProjectExecutor.cpp b/src/graph/executor/query/ProjectExecutor.cpp index 3f54a1daa6b..1fd85a08e64 100644 --- a/src/graph/executor/query/ProjectExecutor.cpp +++ b/src/graph/executor/query/ProjectExecutor.cpp @@ -11,6 +11,7 @@ namespace nebula { namespace graph { folly::Future ProjectExecutor::execute() { + // throw std::bad_alloc in MemoryCheckGuard verified SCOPED_TIMER(&execTime_); auto *project = asNode(node()); auto iter = ectx_->getResult(project->inputVar()).iter(); diff --git a/src/graph/executor/query/ScanEdgesExecutor.cpp b/src/graph/executor/query/ScanEdgesExecutor.cpp index 4262ead8b72..769567620b2 100644 --- a/src/graph/executor/query/ScanEdgesExecutor.cpp +++ b/src/graph/executor/query/ScanEdgesExecutor.cpp @@ -45,12 +45,6 @@ folly::Future ScanEdgesExecutor::scanEdges() { SCOPED_TIMER(&execTime_); addStats(rpcResp, otherStats_); return handleResp(std::move(rpcResp), {}); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/ScanVerticesExecutor.cpp b/src/graph/executor/query/ScanVerticesExecutor.cpp index 9cd80e7011f..ba15ced7412 100644 --- a/src/graph/executor/query/ScanVerticesExecutor.cpp +++ b/src/graph/executor/query/ScanVerticesExecutor.cpp @@ -46,12 +46,6 @@ folly::Future ScanVerticesExecutor::scanVertices() { SCOPED_TIMER(&execTime_); addStats(rpcResp, otherStats_); return handleResp(std::move(rpcResp), sv->colNames()); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc &) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception &e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp index f6cf7910acf..f4b85fd6a8b 100644 --- a/src/graph/executor/query/TraverseExecutor.cpp +++ b/src/graph/executor/query/TraverseExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/query/TraverseExecutor.h" #include "clients/storage/StorageClient.h" +#include "common/memory/MemoryTracker.h" #include "graph/service/GraphFlags.h" #include "graph/util/SchemaUtil.h" #include "graph/util/Utils.h" @@ -100,17 +101,12 @@ folly::Future TraverseExecutor::getNeighbors() { currentStep_ == 1 ? traverse_->tagFilter() : nullptr) .via(runner()) .thenValue([this, getNbrTime](StorageRpcResponse&& resp) mutable { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; vids_.clear(); SCOPED_TIMER(&execTime_); addStats(resp, getNbrTime.elapsedInUSec()); return handleResponse(std::move(resp)); - }) - .thenError( - folly::tag_t{}, - [](const std::bad_alloc&) { return folly::makeFuture(memoryExceededStatus()); }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -310,6 +306,9 @@ folly::Future TraverseExecutor::buildPathMultiJobs(size_t minStep, size_ auto scatter = [this, minStep, maxStep]( size_t begin, size_t end, Iterator* tmpIter) mutable -> std::vector { + // outside caller should already turn on throwOnMemoryExceeded + DCHECK(memory::MemoryTracker::isOn()) << "MemoryTracker is off"; + // MemoryTrackerVerified std::vector rows; for (; tmpIter->valid() && begin++ < end; tmpIter->next()) { auto& initVertex = tmpIter->getColumn(0); @@ -324,6 +323,7 @@ folly::Future TraverseExecutor::buildPathMultiJobs(size_t minStep, size_ }; auto gather = [this](std::vector> resp) mutable -> Status { + // MemoryTrackerVerified memory::MemoryCheckGuard guard; for (auto& rows : resp) { if (rows.empty()) { diff --git a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp index 8e4f17f171e..d2d121378bf 100644 --- a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp +++ b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp @@ -16,15 +16,15 @@ AsyncMsgNotifyBasedScheduler::AsyncMsgNotifyBasedScheduler(QueryContext* qctx) : } folly::Future AsyncMsgNotifyBasedScheduler::schedule() { - auto root = qctx_->plan()->root(); - if (FLAGS_enable_lifetime_optimize) { - // special for root - root->outputVarPtr()->userCount.store(std::numeric_limits::max(), - std::memory_order_relaxed); - analyzeLifetime(root); - } - auto executor = Executor::create(root, qctx_); - return doSchedule(executor); + auto root = qctx_->plan()->root(); + if (FLAGS_enable_lifetime_optimize) { + // special for root + root->outputVarPtr()->userCount.store(std::numeric_limits::max(), + std::memory_order_relaxed); + analyzeLifetime(root); + } + auto executor = Executor::create(root, qctx_); + return doSchedule(executor); } folly::Future AsyncMsgNotifyBasedScheduler::doSchedule(Executor* root) const { @@ -82,6 +82,14 @@ folly::Future AsyncMsgNotifyBasedScheduler::doSchedule(Executor* root) c auto currentExePromises = std::move(currentPromisesFound->second); scheduleExecutor(std::move(currentExeFutures), exe, runner) + // This is the root catch of bad_alloc for Executors, + // all chained returned future is checked here + .thenError( + folly::tag_t{}, + [](const std::bad_alloc&) { + return folly::makeFuture(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + }) .thenTry([this, pros = std::move(currentExePromises)](auto&& t) mutable { // any exception or status not ok handled with notifyError if (t.hasException()) { @@ -135,6 +143,7 @@ folly::Future AsyncMsgNotifyBasedScheduler::runSelect( return execute(select); }) .thenValue([select, this](auto&& selectStatus) mutable -> folly::Future { + memory::MemoryCheckGuard guard; NG_RETURN_IF_ERROR(selectStatus); auto val = qctx_->ectx()->getValue(select->node()->outputVar()); if (!val.isBool()) { @@ -147,45 +156,22 @@ folly::Future AsyncMsgNotifyBasedScheduler::runSelect( return doSchedule(select->thenBody()); } return doSchedule(select->elseBody()); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } folly::Future AsyncMsgNotifyBasedScheduler::runExecutor( std::vector>&& futures, Executor* exe, folly::Executor* runner) const { - return folly::collect(futures) - .via(runner) - .thenValue([exe, this](auto&& t) mutable -> folly::Future { + return folly::collect(futures).via(runner).thenValue( + [exe, this](auto&& t) mutable -> folly::Future { NG_RETURN_IF_ERROR(checkStatus(std::move(t))); // Execute in current thread. return execute(exe); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } folly::Future AsyncMsgNotifyBasedScheduler::runLeafExecutor(Executor* exe, folly::Executor* runner) const { - return std::move(execute(exe)) - .via(runner) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + return std::move(execute(exe)).via(runner); } folly::Future AsyncMsgNotifyBasedScheduler::runLoop( @@ -199,6 +185,7 @@ folly::Future AsyncMsgNotifyBasedScheduler::runLoop( return execute(loop); }) .thenValue([loop, runner, this](auto&& loopStatus) mutable -> folly::Future { + memory::MemoryCheckGuard guard; NG_RETURN_IF_ERROR(loopStatus); auto val = qctx_->ectx()->getValue(loop->node()->outputVar()); if (!val.isBool()) { @@ -212,13 +199,6 @@ folly::Future AsyncMsgNotifyBasedScheduler::runLoop( std::vector> fs; fs.emplace_back(doSchedule(loop->loopBody())); return runLoop(std::move(fs), loop, runner); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); }); } @@ -245,22 +225,16 @@ void AsyncMsgNotifyBasedScheduler::notifyError(std::vector AsyncMsgNotifyBasedScheduler::execute(Executor* executor) const { + memory::MemoryCheckGuard guard1; auto status = executor->open(); if (!status.ok()) { return executor->error(std::move(status)); } - return executor->execute() - .thenValue([executor](Status s) { - NG_RETURN_IF_ERROR(s); - return executor->close(); - }) - .thenError(folly::tag_t{}, - [](const std::bad_alloc&) { - return folly::makeFuture(Executor::memoryExceededStatus()); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - return folly::makeFuture(std::runtime_error(e.what())); - }); + return executor->execute().thenValue([executor](Status s) { + memory::MemoryCheckGuard guard2; + NG_RETURN_IF_ERROR(s); + return executor->close(); + }); } } // namespace graph diff --git a/src/graph/service/QueryInstance.cpp b/src/graph/service/QueryInstance.cpp index 63bd78b7714..9e4d9f6406c 100644 --- a/src/graph/service/QueryInstance.cpp +++ b/src/graph/service/QueryInstance.cpp @@ -67,7 +67,8 @@ void QueryInstance::execute() { .thenError(folly::tag_t{}, [this](const std::exception &e) { onError(Status::Error("%s", e.what())); }); } catch (std::bad_alloc &e) { - onError(Executor::memoryExceededStatus()); + onError(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); } catch (std::exception &e) { onError(Status::Error("%s", e.what())); } catch (...) { @@ -191,6 +192,8 @@ void QueryInstance::onError(Status status) { case Status::Code::kUserNotFound: case Status::Code::kListenerNotFound: case Status::Code::kSessionNotFound: + case Status::Code::kGraphMemoryExceeded: + case Status::Code::kStorageMemoryExceeded: rctx->resp().errorCode = ErrorCode::E_EXECUTION_ERROR; break; } diff --git a/src/kvstore/listener/test/CMakeLists.txt b/src/kvstore/listener/test/CMakeLists.txt index 1041cfd17e9..09ce4aba2ae 100644 --- a/src/kvstore/listener/test/CMakeLists.txt +++ b/src/kvstore/listener/test/CMakeLists.txt @@ -23,6 +23,7 @@ set(LISTENER_TEST_LIBS $ $ $ + $ $ $ $ diff --git a/src/kvstore/raftex/test/CMakeLists.txt b/src/kvstore/raftex/test/CMakeLists.txt index fc47919be15..4d0d61e8d36 100644 --- a/src/kvstore/raftex/test/CMakeLists.txt +++ b/src/kvstore/raftex/test/CMakeLists.txt @@ -14,6 +14,7 @@ set(RAFTEX_TEST_LIBS $ $ $ + $ $ $ $ diff --git a/src/kvstore/test/CMakeLists.txt b/src/kvstore/test/CMakeLists.txt index 5db2e891e36..129995acbfb 100644 --- a/src/kvstore/test/CMakeLists.txt +++ b/src/kvstore/test/CMakeLists.txt @@ -23,6 +23,7 @@ set(KVSTORE_TEST_LIBS $ $ $ + $ $ $ $ diff --git a/src/kvstore/wal/test/CMakeLists.txt b/src/kvstore/wal/test/CMakeLists.txt index bb4c77976d6..263afd373c3 100644 --- a/src/kvstore/wal/test/CMakeLists.txt +++ b/src/kvstore/wal/test/CMakeLists.txt @@ -10,6 +10,7 @@ set(WAL_TEST_LIBS $ $ $ + $ $ $ $ diff --git a/src/storage/BaseProcessor.h b/src/storage/BaseProcessor.h index 8dd2975cf7a..b94fc723f08 100644 --- a/src/storage/BaseProcessor.h +++ b/src/storage/BaseProcessor.h @@ -30,11 +30,14 @@ class BaseProcessor { virtual ~BaseProcessor() = default; + void memoryExceeded() { + memoryExceeded_ = true; + } + folly::Future getFuture() { return promise_.getFuture(); } - protected: virtual void onFinished() { if (counters_) { stats::StatsManager::addValue(counters_->numCalls_); @@ -83,6 +86,7 @@ class BaseProcessor { delete this; } + protected: nebula::cpp2::ErrorCode getSpaceVidLen(GraphSpaceID spaceId) { auto len = this->env_->schemaMan_->getSpaceVidLen(spaceId); if (!len.ok()) { @@ -160,6 +164,31 @@ class BaseProcessor { bool memoryExceeded_{false}; }; +/// Helper class wrap the passed in Func in a MemoryTracker turned on scope. +template +struct MemoryCheckScope { + MemoryCheckScope(BaseProcessor* processor, Func f) + : processor_(processor), f_(std::move(f)) {} + + ~MemoryCheckScope() { + memory::MemoryCheckGuard guard; + try { + f_(); + } catch (std::bad_alloc& e) { + processor_->memoryExceeded(); + processor_->onError(); + } catch (std::exception& e) { + LOG(ERROR) << e.what(); + processor_->onError(); + } catch (...) { + processor_->onError(); + } + } + + BaseProcessor* processor_; + Func f_; +}; + } // namespace storage } // namespace nebula diff --git a/src/storage/GraphStorageServiceHandler.cpp b/src/storage/GraphStorageServiceHandler.cpp index d9e6688f0c0..7d0d18971d5 100644 --- a/src/storage/GraphStorageServiceHandler.cpp +++ b/src/storage/GraphStorageServiceHandler.cpp @@ -26,10 +26,25 @@ #include "storage/transaction/ChainDeleteEdgesGroupProcessor.h" #include "storage/transaction/ChainUpdateEdgeLocalProcessor.h" +// Processor::process's root memory check is turn on here. +// if the call stack in current thread, +// Processors DO NOT NEED handle error in their logic. +// else (do some work in another thread) +// Processors need handle error in that thread by itself #define RETURN_FUTURE(processor) \ memory::MemoryCheckGuard guard; \ auto f = processor->getFuture(); \ - processor->process(req); \ + try { \ + processor->process(req); \ + } catch (std::bad_alloc & e) { \ + processor->memoryExceeded(); \ + processor->onError(); \ + } catch (std::exception & e) { \ + LOG(ERROR) << e.what(); \ + processor->onError(); \ + } catch (...) { \ + processor->onError(); \ + } \ return f; namespace nebula { diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 12c2e239cca..975cc348794 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -28,20 +28,11 @@ ProcessorCounters kLookupCounters; // print Plan for debug inline void printPlan(IndexNode* node, int tab = 0); void LookupProcessor::process(const cpp2::LookupIndexRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/kv/GetProcessor.cpp b/src/storage/kv/GetProcessor.cpp index 22f507cec94..97a51efc1d4 100644 --- a/src/storage/kv/GetProcessor.cpp +++ b/src/storage/kv/GetProcessor.cpp @@ -13,53 +13,42 @@ namespace storage { ProcessorCounters kGetCounters; void GetProcessor::process(const cpp2::KVGetRequest& req) { - try { - CHECK_NOTNULL(env_->kvstore_); - GraphSpaceID spaceId = req.get_space_id(); - bool returnPartly = req.get_return_partly(); - - std::unordered_map pairs; - size_t size = 0; - for (auto& part : req.get_parts()) { - size += part.second.size(); - } - pairs.reserve(size); - - for (auto& part : req.get_parts()) { - auto partId = part.first; - auto& keys = part.second; - std::vector kvKeys; - kvKeys.reserve(part.second.size()); - std::transform( - keys.begin(), keys.end(), std::back_inserter(kvKeys), [partId](const auto& key) { - return NebulaKeyUtils::kvKey(partId, key); - }); - std::vector values; - auto ret = env_->kvstore_->multiGet(spaceId, partId, kvKeys, &values); - if ((ret.first == nebula::cpp2::ErrorCode::SUCCEEDED) || - (ret.first == nebula::cpp2::ErrorCode::E_PARTIAL_RESULT && returnPartly)) { - auto& status = ret.second; - for (size_t i = 0; i < kvKeys.size(); i++) { - if (status[i].ok()) { - pairs.emplace(keys[i], values[i]); - } + CHECK_NOTNULL(env_->kvstore_); + GraphSpaceID spaceId = req.get_space_id(); + bool returnPartly = req.get_return_partly(); + + std::unordered_map pairs; + size_t size = 0; + for (auto& part : req.get_parts()) { + size += part.second.size(); + } + pairs.reserve(size); + + for (auto& part : req.get_parts()) { + auto partId = part.first; + auto& keys = part.second; + std::vector kvKeys; + kvKeys.reserve(part.second.size()); + std::transform(keys.begin(), keys.end(), std::back_inserter(kvKeys), [partId](const auto& key) { + return NebulaKeyUtils::kvKey(partId, key); + }); + std::vector values; + auto ret = env_->kvstore_->multiGet(spaceId, partId, kvKeys, &values); + if ((ret.first == nebula::cpp2::ErrorCode::SUCCEEDED) || + (ret.first == nebula::cpp2::ErrorCode::E_PARTIAL_RESULT && returnPartly)) { + auto& status = ret.second; + for (size_t i = 0; i < kvKeys.size(); i++) { + if (status[i].ok()) { + pairs.emplace(keys[i], values[i]); } - } else { - handleErrorCode(ret.first, spaceId, partId); } + } else { + handleErrorCode(ret.first, spaceId, partId); } - - resp_.key_values_ref() = std::move(pairs); - this->onFinished(); - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); } + + resp_.key_values_ref() = std::move(pairs); + this->onFinished(); } } // namespace storage diff --git a/src/storage/kv/PutProcessor.cpp b/src/storage/kv/PutProcessor.cpp index 4f97feada50..24ef9a9dd25 100644 --- a/src/storage/kv/PutProcessor.cpp +++ b/src/storage/kv/PutProcessor.cpp @@ -13,29 +13,19 @@ namespace storage { ProcessorCounters kPutCounters; void PutProcessor::process(const cpp2::KVPutRequest& req) { - try { - CHECK_NOTNULL(env_->kvstore_); - const auto& pairs = req.get_parts(); - auto space = req.get_space_id(); - callingNum_ = pairs.size(); + CHECK_NOTNULL(env_->kvstore_); + const auto& pairs = req.get_parts(); + auto space = req.get_space_id(); + callingNum_ = pairs.size(); - std::for_each(pairs.begin(), pairs.end(), [&](auto& value) { - auto part = value.first; - std::vector data; - for (auto& pair : value.second) { - data.emplace_back(std::move(NebulaKeyUtils::kvKey(part, pair.key)), std::move(pair.value)); - } - doPut(space, part, std::move(data)); - }); - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); - } + std::for_each(pairs.begin(), pairs.end(), [&](auto& value) { + auto part = value.first; + std::vector data; + for (auto& pair : value.second) { + data.emplace_back(std::move(NebulaKeyUtils::kvKey(part, pair.key)), std::move(pair.value)); + } + doPut(space, part, std::move(data)); + }); } } // namespace storage diff --git a/src/storage/kv/RemoveProcessor.cpp b/src/storage/kv/RemoveProcessor.cpp index f074de0b123..3588e5b95e6 100644 --- a/src/storage/kv/RemoveProcessor.cpp +++ b/src/storage/kv/RemoveProcessor.cpp @@ -13,29 +13,19 @@ namespace storage { ProcessorCounters kRemoveCounters; void RemoveProcessor::process(const cpp2::KVRemoveRequest& req) { - try { - CHECK_NOTNULL(env_->kvstore_); - const auto& pairs = req.get_parts(); - auto space = req.get_space_id(); - callingNum_ = pairs.size(); + CHECK_NOTNULL(env_->kvstore_); + const auto& pairs = req.get_parts(); + auto space = req.get_space_id(); + callingNum_ = pairs.size(); - std::for_each(pairs.begin(), pairs.end(), [&](auto& value) { - auto part = value.first; - std::vector keys; - for (auto& key : value.second) { - keys.emplace_back(std::move(NebulaKeyUtils::kvKey(part, key))); - } - doRemove(space, part, std::move(keys)); - }); - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); - } + std::for_each(pairs.begin(), pairs.end(), [&](auto& value) { + auto part = value.first; + std::vector keys; + for (auto& key : value.second) { + keys.emplace_back(std::move(NebulaKeyUtils::kvKey(part, key))); + } + doRemove(space, part, std::move(keys)); + }); } } // namespace storage diff --git a/src/storage/mutate/AddEdgesProcessor.cpp b/src/storage/mutate/AddEdgesProcessor.cpp index 47ca51a33cc..9bd0a3d21ab 100644 --- a/src/storage/mutate/AddEdgesProcessor.cpp +++ b/src/storage/mutate/AddEdgesProcessor.cpp @@ -21,53 +21,43 @@ namespace storage { ProcessorCounters kAddEdgesCounters; void AddEdgesProcessor::process(const cpp2::AddEdgesRequest& req) { - try { - spaceId_ = req.get_space_id(); - ifNotExists_ = req.get_if_not_exists(); - const auto& partEdges = req.get_parts(); - - CHECK_NOTNULL(env_->schemaMan_); - auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); - if (!ret.ok()) { - LOG(ERROR) << ret.status(); - for (auto& part : partEdges) { - pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); - } - onFinished(); - return; + spaceId_ = req.get_space_id(); + ifNotExists_ = req.get_if_not_exists(); + const auto& partEdges = req.get_parts(); + + CHECK_NOTNULL(env_->schemaMan_); + auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); + if (!ret.ok()) { + LOG(ERROR) << ret.status(); + for (auto& part : partEdges) { + pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); } + onFinished(); + return; + } - spaceVidLen_ = ret.value(); - callingNum_ = partEdges.size(); + spaceVidLen_ = ret.value(); + callingNum_ = partEdges.size(); - CHECK_NOTNULL(env_->indexMan_); - auto iRet = env_->indexMan_->getEdgeIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << iRet.status(); - for (auto& part : partEdges) { - pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); - } - onFinished(); - return; + CHECK_NOTNULL(env_->indexMan_); + auto iRet = env_->indexMan_->getEdgeIndexes(spaceId_); + if (!iRet.ok()) { + LOG(ERROR) << iRet.status(); + for (auto& part : partEdges) { + pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); } - indexes_ = std::move(iRet).value(); - ignoreExistedIndex_ = req.get_ignore_existed_index(); + onFinished(); + return; + } + indexes_ = std::move(iRet).value(); + ignoreExistedIndex_ = req.get_ignore_existed_index(); - CHECK_NOTNULL(env_->kvstore_); + CHECK_NOTNULL(env_->kvstore_); - if (indexes_.empty()) { - doProcess(req); - } else { - doProcessWithIndex(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (indexes_.empty()) { + doProcess(req); + } else { + doProcessWithIndex(req); } } diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp index 22a94e2948e..0cdd04b769a 100644 --- a/src/storage/mutate/AddVerticesProcessor.cpp +++ b/src/storage/mutate/AddVerticesProcessor.cpp @@ -22,51 +22,40 @@ namespace storage { ProcessorCounters kAddVerticesCounters; void AddVerticesProcessor::process(const cpp2::AddVerticesRequest& req) { - try { - memory::MemoryCheckGuard guard; - spaceId_ = req.get_space_id(); - const auto& partVertices = req.get_parts(); - ifNotExists_ = req.get_if_not_exists(); - CHECK_NOTNULL(env_->schemaMan_); - auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); - if (!ret.ok()) { - LOG(ERROR) << ret.status(); - for (auto& part : partVertices) { - pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); - } - onFinished(); - return; + spaceId_ = req.get_space_id(); + const auto& partVertices = req.get_parts(); + ifNotExists_ = req.get_if_not_exists(); + CHECK_NOTNULL(env_->schemaMan_); + auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); + if (!ret.ok()) { + LOG(ERROR) << ret.status(); + for (auto& part : partVertices) { + pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); } - spaceVidLen_ = ret.value(); - callingNum_ = partVertices.size(); - - CHECK_NOTNULL(env_->indexMan_); - auto iRet = env_->indexMan_->getTagIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << iRet.status(); - for (auto& part : partVertices) { - pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); - } - onFinished(); - return; + onFinished(); + return; + } + spaceVidLen_ = ret.value(); + callingNum_ = partVertices.size(); + + CHECK_NOTNULL(env_->indexMan_); + auto iRet = env_->indexMan_->getTagIndexes(spaceId_); + if (!iRet.ok()) { + LOG(ERROR) << iRet.status(); + for (auto& part : partVertices) { + pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); } - indexes_ = std::move(iRet).value(); - ignoreExistedIndex_ = req.get_ignore_existed_index(); + onFinished(); + return; + } + indexes_ = std::move(iRet).value(); + ignoreExistedIndex_ = req.get_ignore_existed_index(); - CHECK_NOTNULL(env_->kvstore_); - if (indexes_.empty()) { - doProcess(req); - } else { - doProcessWithIndex(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + CHECK_NOTNULL(env_->kvstore_); + if (indexes_.empty()) { + doProcess(req); + } else { + doProcessWithIndex(req); } } diff --git a/src/storage/mutate/DeleteEdgesProcessor.cpp b/src/storage/mutate/DeleteEdgesProcessor.cpp index eacfe721c6b..944e25bd69a 100644 --- a/src/storage/mutate/DeleteEdgesProcessor.cpp +++ b/src/storage/mutate/DeleteEdgesProcessor.cpp @@ -20,153 +20,140 @@ namespace storage { ProcessorCounters kDelEdgesCounters; void DeleteEdgesProcessor::process(const cpp2::DeleteEdgesRequest& req) { - try { - spaceId_ = req.get_space_id(); - const auto& partEdges = req.get_parts(); - - CHECK_NOTNULL(env_->schemaMan_); - auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); - if (!ret.ok()) { - LOG(ERROR) << ret.status(); - for (auto& part : partEdges) { - pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); - } - onFinished(); - return; + spaceId_ = req.get_space_id(); + const auto& partEdges = req.get_parts(); + + CHECK_NOTNULL(env_->schemaMan_); + auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); + if (!ret.ok()) { + LOG(ERROR) << ret.status(); + for (auto& part : partEdges) { + pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); } - spaceVidLen_ = ret.value(); - callingNum_ = partEdges.size(); - - CHECK_NOTNULL(env_->indexMan_); - auto iRet = env_->indexMan_->getEdgeIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << iRet.status(); - for (auto& part : partEdges) { - pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); - } - onFinished(); - return; + onFinished(); + return; + } + spaceVidLen_ = ret.value(); + callingNum_ = partEdges.size(); + + CHECK_NOTNULL(env_->indexMan_); + auto iRet = env_->indexMan_->getEdgeIndexes(spaceId_); + if (!iRet.ok()) { + LOG(ERROR) << iRet.status(); + for (auto& part : partEdges) { + pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); } - indexes_ = std::move(iRet).value(); - - CHECK_NOTNULL(env_->kvstore_); - if (indexes_.empty()) { - // Operate every part, the graph layer guarantees the unique of the edgeKey - for (auto& part : partEdges) { - std::vector keys; - keys.reserve(32); - auto partId = part.first; - auto code = nebula::cpp2::ErrorCode::SUCCEEDED; - for (auto& edgeKey : part.second) { - if (!NebulaKeyUtils::isValidVidLen( - spaceVidLen_, edgeKey.src_ref()->getStr(), edgeKey.dst_ref()->getStr())) { - LOG(ERROR) << "Space " << spaceId_ << " vertex length invalid, " - << "space vid len: " << spaceVidLen_ - << ", edge srcVid: " << *edgeKey.src_ref() - << " dstVid: " << *edgeKey.dst_ref(); - code = nebula::cpp2::ErrorCode::E_INVALID_VID; - break; - } - // todo(doodle): delete lock in toss - auto edge = NebulaKeyUtils::edgeKey(spaceVidLen_, - partId, - edgeKey.src_ref()->getStr(), - *edgeKey.edge_type_ref(), - *edgeKey.ranking_ref(), - edgeKey.dst_ref()->getStr()); - keys.emplace_back(edge.data(), edge.size()); - } - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleAsync(spaceId_, partId, code); - continue; + onFinished(); + return; + } + indexes_ = std::move(iRet).value(); + + CHECK_NOTNULL(env_->kvstore_); + if (indexes_.empty()) { + // Operate every part, the graph layer guarantees the unique of the edgeKey + for (auto& part : partEdges) { + std::vector keys; + keys.reserve(32); + auto partId = part.first; + auto code = nebula::cpp2::ErrorCode::SUCCEEDED; + for (auto& edgeKey : part.second) { + if (!NebulaKeyUtils::isValidVidLen( + spaceVidLen_, edgeKey.src_ref()->getStr(), edgeKey.dst_ref()->getStr())) { + LOG(ERROR) << "Space " << spaceId_ << " vertex length invalid, " + << "space vid len: " << spaceVidLen_ << ", edge srcVid: " << *edgeKey.src_ref() + << " dstVid: " << *edgeKey.dst_ref(); + code = nebula::cpp2::ErrorCode::E_INVALID_VID; + break; } + // todo(doodle): delete lock in toss + auto edge = NebulaKeyUtils::edgeKey(spaceVidLen_, + partId, + edgeKey.src_ref()->getStr(), + *edgeKey.edge_type_ref(), + *edgeKey.ranking_ref(), + edgeKey.dst_ref()->getStr()); + keys.emplace_back(edge.data(), edge.size()); + } + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleAsync(spaceId_, partId, code); + continue; + } - HookFuncPara para; - if (tossHookFunc_) { - para.keys.emplace(&keys); - (*tossHookFunc_)(para); - } - if (para.result) { - env_->kvstore_->asyncAppendBatch( - spaceId_, - partId, - std::move(para.result.value()), - [partId, this](nebula::cpp2::ErrorCode rc) { handleAsync(spaceId_, partId, rc); }); - } else { - doRemove(spaceId_, partId, std::move(keys)); - stats::StatsManager::addValue(kNumEdgesDeleted, keys.size()); - } + HookFuncPara para; + if (tossHookFunc_) { + para.keys.emplace(&keys); + (*tossHookFunc_)(para); } - } else { - for (auto& part : partEdges) { - IndexCountWrapper wrapper(env_); - auto partId = part.first; - std::vector dummyLock; - dummyLock.reserve(part.second.size()); - - nebula::cpp2::ErrorCode err = nebula::cpp2::ErrorCode::SUCCEEDED; - for (const auto& edgeKey : part.second) { - if (!NebulaKeyUtils::isValidVidLen( - spaceVidLen_, edgeKey.src_ref()->getStr(), edgeKey.dst_ref()->getStr())) { - LOG(ERROR) << "Space " << spaceId_ << " vertex length invalid, " - << "space vid len: " << spaceVidLen_ - << ", edge srcVid: " << *edgeKey.src_ref() - << " dstVid: " << *edgeKey.dst_ref(); - err = nebula::cpp2::ErrorCode::E_INVALID_VID; - break; - } - auto l = std::make_tuple(spaceId_, - partId, - edgeKey.src_ref()->getStr(), - *edgeKey.edge_type_ref(), - *edgeKey.ranking_ref(), - edgeKey.dst_ref()->getStr()); - if (std::find(dummyLock.begin(), dummyLock.end(), l) == dummyLock.end()) { - if (!env_->edgesML_->try_lock(l)) { - LOG(ERROR) << folly::sformat("The edge locked : src {}, type {}, tank {}, dst {}", - edgeKey.src_ref()->getStr(), - *edgeKey.edge_type_ref(), - *edgeKey.ranking_ref(), - edgeKey.dst_ref()->getStr()); - err = nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; - break; - } - dummyLock.emplace_back(std::move(l)); - } - } - if (err != nebula::cpp2::ErrorCode::SUCCEEDED) { - env_->edgesML_->unlockBatch(dummyLock); - handleAsync(spaceId_, partId, err); - continue; - } - auto batch = deleteEdges(partId, std::move(part.second)); - if (!nebula::ok(batch)) { - env_->edgesML_->unlockBatch(dummyLock); - handleAsync(spaceId_, partId, nebula::error(batch)); - continue; - } - DCHECK(!nebula::value(batch).empty()); - nebula::MemoryLockGuard lg(env_->edgesML_.get(), std::move(dummyLock), false, false); + if (para.result) { env_->kvstore_->asyncAppendBatch( spaceId_, partId, - std::move(nebula::value(batch)), - [l = std::move(lg), icw = std::move(wrapper), partId, this]( - nebula::cpp2::ErrorCode code) { - UNUSED(l); - UNUSED(icw); - handleAsync(spaceId_, partId, code); - }); + std::move(para.result.value()), + [partId, this](nebula::cpp2::ErrorCode rc) { handleAsync(spaceId_, partId, rc); }); + } else { + doRemove(spaceId_, partId, std::move(keys)); + stats::StatsManager::addValue(kNumEdgesDeleted, keys.size()); + } + } + } else { + for (auto& part : partEdges) { + IndexCountWrapper wrapper(env_); + auto partId = part.first; + std::vector dummyLock; + dummyLock.reserve(part.second.size()); + + nebula::cpp2::ErrorCode err = nebula::cpp2::ErrorCode::SUCCEEDED; + for (const auto& edgeKey : part.second) { + if (!NebulaKeyUtils::isValidVidLen( + spaceVidLen_, edgeKey.src_ref()->getStr(), edgeKey.dst_ref()->getStr())) { + LOG(ERROR) << "Space " << spaceId_ << " vertex length invalid, " + << "space vid len: " << spaceVidLen_ << ", edge srcVid: " << *edgeKey.src_ref() + << " dstVid: " << *edgeKey.dst_ref(); + err = nebula::cpp2::ErrorCode::E_INVALID_VID; + break; + } + auto l = std::make_tuple(spaceId_, + partId, + edgeKey.src_ref()->getStr(), + *edgeKey.edge_type_ref(), + *edgeKey.ranking_ref(), + edgeKey.dst_ref()->getStr()); + if (std::find(dummyLock.begin(), dummyLock.end(), l) == dummyLock.end()) { + if (!env_->edgesML_->try_lock(l)) { + LOG(ERROR) << folly::sformat("The edge locked : src {}, type {}, tank {}, dst {}", + edgeKey.src_ref()->getStr(), + *edgeKey.edge_type_ref(), + *edgeKey.ranking_ref(), + edgeKey.dst_ref()->getStr()); + err = nebula::cpp2::ErrorCode::E_DATA_CONFLICT_ERROR; + break; + } + dummyLock.emplace_back(std::move(l)); + } + } + if (err != nebula::cpp2::ErrorCode::SUCCEEDED) { + env_->edgesML_->unlockBatch(dummyLock); + handleAsync(spaceId_, partId, err); + continue; + } + auto batch = deleteEdges(partId, std::move(part.second)); + if (!nebula::ok(batch)) { + env_->edgesML_->unlockBatch(dummyLock); + handleAsync(spaceId_, partId, nebula::error(batch)); + continue; } + DCHECK(!nebula::value(batch).empty()); + nebula::MemoryLockGuard lg(env_->edgesML_.get(), std::move(dummyLock), false, false); + env_->kvstore_->asyncAppendBatch(spaceId_, + partId, + std::move(nebula::value(batch)), + [l = std::move(lg), icw = std::move(wrapper), partId, this]( + nebula::cpp2::ErrorCode code) { + UNUSED(l); + UNUSED(icw); + handleAsync(spaceId_, partId, code); + }); } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); } } diff --git a/src/storage/mutate/DeleteTagsProcessor.cpp b/src/storage/mutate/DeleteTagsProcessor.cpp index 9b189fef728..7ffc6f28696 100644 --- a/src/storage/mutate/DeleteTagsProcessor.cpp +++ b/src/storage/mutate/DeleteTagsProcessor.cpp @@ -18,87 +18,76 @@ namespace storage { ProcessorCounters kDelTagsCounters; void DeleteTagsProcessor::process(const cpp2::DeleteTagsRequest& req) { - try { - spaceId_ = req.get_space_id(); - const auto& parts = req.get_parts(); + spaceId_ = req.get_space_id(); + const auto& parts = req.get_parts(); - CHECK_NOTNULL(env_->schemaMan_); - auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); - if (!ret.ok()) { - LOG(ERROR) << ret.status(); - for (auto& part : parts) { - pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); - } - onFinished(); - return; + CHECK_NOTNULL(env_->schemaMan_); + auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); + if (!ret.ok()) { + LOG(ERROR) << ret.status(); + for (auto& part : parts) { + pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); } - spaceVidLen_ = ret.value(); - callingNum_ = parts.size(); + onFinished(); + return; + } + spaceVidLen_ = ret.value(); + callingNum_ = parts.size(); - CHECK_NOTNULL(env_->indexMan_); - auto iRet = env_->indexMan_->getTagIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << iRet.status(); - for (auto& part : parts) { - pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); - } - onFinished(); - return; + CHECK_NOTNULL(env_->indexMan_); + auto iRet = env_->indexMan_->getTagIndexes(spaceId_); + if (!iRet.ok()) { + LOG(ERROR) << iRet.status(); + for (auto& part : parts) { + pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); } - indexes_ = std::move(iRet).value(); + onFinished(); + return; + } + indexes_ = std::move(iRet).value(); - CHECK_NOTNULL(env_->kvstore_); - if (indexes_.empty()) { - std::vector keys; - keys.reserve(32); - for (const auto& part : parts) { - auto partId = part.first; - const auto& delTags = part.second; - keys.clear(); - for (const auto& entry : delTags) { - const auto& vId = entry.get_id().getStr(); - for (const auto& tagId : entry.get_tags()) { - auto key = NebulaKeyUtils::tagKey(spaceVidLen_, partId, vId, tagId); - keys.emplace_back(std::move(key)); - } + CHECK_NOTNULL(env_->kvstore_); + if (indexes_.empty()) { + std::vector keys; + keys.reserve(32); + for (const auto& part : parts) { + auto partId = part.first; + const auto& delTags = part.second; + keys.clear(); + for (const auto& entry : delTags) { + const auto& vId = entry.get_id().getStr(); + for (const auto& tagId : entry.get_tags()) { + auto key = NebulaKeyUtils::tagKey(spaceVidLen_, partId, vId, tagId); + keys.emplace_back(std::move(key)); } - doRemove(spaceId_, partId, std::move(keys)); - stats::StatsManager::addValue(kNumTagsDeleted, keys.size()); } - } else { - for (const auto& part : parts) { - IndexCountWrapper wrapper(env_); - auto partId = part.first; - std::vector lockedKeys; - auto batch = deleteTags(partId, part.second, lockedKeys); - if (!nebula::ok(batch)) { - env_->verticesML_->unlockBatch(lockedKeys); - handleAsync(spaceId_, partId, nebula::error(batch)); - continue; - } - // keys has been locked in deleteTags - nebula::MemoryLockGuard lg( - env_->verticesML_.get(), std::move(lockedKeys), false, false); - env_->kvstore_->asyncAppendBatch( - spaceId_, - partId, - std::move(nebula::value(batch)), - [l = std::move(lg), icw = std::move(wrapper), partId, this]( - nebula::cpp2::ErrorCode code) { - UNUSED(l); - UNUSED(icw); - handleAsync(spaceId_, partId, code); - }); + doRemove(spaceId_, partId, std::move(keys)); + stats::StatsManager::addValue(kNumTagsDeleted, keys.size()); + } + } else { + for (const auto& part : parts) { + IndexCountWrapper wrapper(env_); + auto partId = part.first; + std::vector lockedKeys; + auto batch = deleteTags(partId, part.second, lockedKeys); + if (!nebula::ok(batch)) { + env_->verticesML_->unlockBatch(lockedKeys); + handleAsync(spaceId_, partId, nebula::error(batch)); + continue; } + // keys has been locked in deleteTags + nebula::MemoryLockGuard lg( + env_->verticesML_.get(), std::move(lockedKeys), false, false); + env_->kvstore_->asyncAppendBatch(spaceId_, + partId, + std::move(nebula::value(batch)), + [l = std::move(lg), icw = std::move(wrapper), partId, this]( + nebula::cpp2::ErrorCode code) { + UNUSED(l); + UNUSED(icw); + handleAsync(spaceId_, partId, code); + }); } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); } } diff --git a/src/storage/mutate/DeleteVerticesProcessor.cpp b/src/storage/mutate/DeleteVerticesProcessor.cpp index 3dbc6ae7060..63a21f2ad20 100644 --- a/src/storage/mutate/DeleteVerticesProcessor.cpp +++ b/src/storage/mutate/DeleteVerticesProcessor.cpp @@ -17,107 +17,95 @@ namespace storage { ProcessorCounters kDelVerticesCounters; void DeleteVerticesProcessor::process(const cpp2::DeleteVerticesRequest& req) { - try { - spaceId_ = req.get_space_id(); - const auto& partVertices = req.get_parts(); + spaceId_ = req.get_space_id(); + const auto& partVertices = req.get_parts(); - CHECK_NOTNULL(env_->schemaMan_); - auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); - if (!ret.ok()) { - LOG(ERROR) << ret.status(); - for (auto& part : partVertices) { - pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); - } - onFinished(); - return; + CHECK_NOTNULL(env_->schemaMan_); + auto ret = env_->schemaMan_->getSpaceVidLen(spaceId_); + if (!ret.ok()) { + LOG(ERROR) << ret.status(); + for (auto& part : partVertices) { + pushResultCode(nebula::cpp2::ErrorCode::E_INVALID_SPACEVIDLEN, part.first); } - spaceVidLen_ = ret.value(); - callingNum_ = partVertices.size(); + onFinished(); + return; + } + spaceVidLen_ = ret.value(); + callingNum_ = partVertices.size(); - CHECK_NOTNULL(env_->indexMan_); - auto iRet = env_->indexMan_->getTagIndexes(spaceId_); - if (!iRet.ok()) { - LOG(ERROR) << iRet.status(); - for (auto& part : partVertices) { - pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); - } - onFinished(); - return; + CHECK_NOTNULL(env_->indexMan_); + auto iRet = env_->indexMan_->getTagIndexes(spaceId_); + if (!iRet.ok()) { + LOG(ERROR) << iRet.status(); + for (auto& part : partVertices) { + pushResultCode(nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND, part.first); } - indexes_ = std::move(iRet).value(); + onFinished(); + return; + } + indexes_ = std::move(iRet).value(); - CHECK_NOTNULL(env_->kvstore_); - if (indexes_.empty()) { - // Operate every part, the graph layer guarantees the unique of the vid - std::vector keys; - keys.reserve(32); - for (auto& part : partVertices) { - auto partId = part.first; - const auto& vertexIds = part.second; - keys.clear(); - auto code = nebula::cpp2::ErrorCode::SUCCEEDED; - for (auto& vid : vertexIds) { - if (!NebulaKeyUtils::isValidVidLen(spaceVidLen_, vid.getStr())) { - LOG(ERROR) << "Space " << spaceId_ << ", vertex length invalid, " - << " space vid len: " << spaceVidLen_ << ", vid is " << vid; - code = nebula::cpp2::ErrorCode::E_INVALID_VID; - break; - } - keys.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid.getStr())); - auto prefix = NebulaKeyUtils::tagPrefix(spaceVidLen_, partId, vid.getStr()); - std::unique_ptr iter; - code = env_->kvstore_->prefix(spaceId_, partId, prefix, &iter); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - VLOG(3) << "Error! ret = " << static_cast(code) << ", spaceID " << spaceId_; - break; - } - while (iter->valid()) { - auto key = iter->key(); - keys.emplace_back(key.str()); - iter->next(); - } + CHECK_NOTNULL(env_->kvstore_); + if (indexes_.empty()) { + // Operate every part, the graph layer guarantees the unique of the vid + std::vector keys; + keys.reserve(32); + for (auto& part : partVertices) { + auto partId = part.first; + const auto& vertexIds = part.second; + keys.clear(); + auto code = nebula::cpp2::ErrorCode::SUCCEEDED; + for (auto& vid : vertexIds) { + if (!NebulaKeyUtils::isValidVidLen(spaceVidLen_, vid.getStr())) { + LOG(ERROR) << "Space " << spaceId_ << ", vertex length invalid, " + << " space vid len: " << spaceVidLen_ << ", vid is " << vid; + code = nebula::cpp2::ErrorCode::E_INVALID_VID; + break; } + keys.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid.getStr())); + auto prefix = NebulaKeyUtils::tagPrefix(spaceVidLen_, partId, vid.getStr()); + std::unique_ptr iter; + code = env_->kvstore_->prefix(spaceId_, partId, prefix, &iter); if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleAsync(spaceId_, partId, code); - continue; + VLOG(3) << "Error! ret = " << static_cast(code) << ", spaceID " << spaceId_; + break; } - doRemove(spaceId_, partId, std::move(keys)); - stats::StatsManager::addValue(kNumVerticesDeleted, keys.size()); - } - } else { - for (auto& pv : partVertices) { - IndexCountWrapper wrapper(env_); - auto partId = pv.first; - std::vector dummyLock; - auto batch = deleteVertices(partId, std::move(pv).second, dummyLock); - if (!nebula::ok(batch)) { - env_->verticesML_->unlockBatch(dummyLock); - handleAsync(spaceId_, partId, nebula::error(batch)); - continue; + while (iter->valid()) { + auto key = iter->key(); + keys.emplace_back(key.str()); + iter->next(); } - DCHECK(!nebula::value(batch).empty()); - nebula::MemoryLockGuard lg( - env_->verticesML_.get(), std::move(dummyLock), false, false); - env_->kvstore_->asyncAppendBatch( - spaceId_, - partId, - std::move(nebula::value(batch)), - [l = std::move(lg), icw = std::move(wrapper), partId, this]( - nebula::cpp2::ErrorCode code) { - UNUSED(l); - UNUSED(icw); - handleAsync(spaceId_, partId, code); - }); } + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleAsync(spaceId_, partId, code); + continue; + } + doRemove(spaceId_, partId, std::move(keys)); + stats::StatsManager::addValue(kNumVerticesDeleted, keys.size()); + } + } else { + for (auto& pv : partVertices) { + IndexCountWrapper wrapper(env_); + auto partId = pv.first; + std::vector dummyLock; + auto batch = deleteVertices(partId, std::move(pv).second, dummyLock); + if (!nebula::ok(batch)) { + env_->verticesML_->unlockBatch(dummyLock); + handleAsync(spaceId_, partId, nebula::error(batch)); + continue; + } + DCHECK(!nebula::value(batch).empty()); + nebula::MemoryLockGuard lg(env_->verticesML_.get(), std::move(dummyLock), false, false); + env_->kvstore_->asyncAppendBatch(spaceId_, + partId, + std::move(nebula::value(batch)), + [l = std::move(lg), icw = std::move(wrapper), partId, this]( + nebula::cpp2::ErrorCode code) { + UNUSED(l); + UNUSED(icw); + handleAsync(spaceId_, partId, code); + }); } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); } } diff --git a/src/storage/mutate/UpdateEdgeProcessor.cpp b/src/storage/mutate/UpdateEdgeProcessor.cpp index 6e6d1db6d2a..599f4e20937 100644 --- a/src/storage/mutate/UpdateEdgeProcessor.cpp +++ b/src/storage/mutate/UpdateEdgeProcessor.cpp @@ -19,23 +19,11 @@ namespace storage { ProcessorCounters kUpdateEdgeCounters; void UpdateEdgeProcessor::process(const cpp2::UpdateEdgeRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { - memory::MemoryCheckGuard guard; - this->doProcess(req); - }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/mutate/UpdateVertexProcessor.cpp b/src/storage/mutate/UpdateVertexProcessor.cpp index 757adb9d6d0..c5fabcff81d 100644 --- a/src/storage/mutate/UpdateVertexProcessor.cpp +++ b/src/storage/mutate/UpdateVertexProcessor.cpp @@ -19,23 +19,11 @@ namespace storage { ProcessorCounters kUpdateVertexCounters; void UpdateVertexProcessor::process(const cpp2::UpdateVertexRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { - memory::MemoryCheckGuard guard; - this->doProcess(req); - }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/query/GetDstBySrcProcessor.cpp b/src/storage/query/GetDstBySrcProcessor.cpp index 11240738108..f4a1ce38029 100644 --- a/src/storage/query/GetDstBySrcProcessor.cpp +++ b/src/storage/query/GetDstBySrcProcessor.cpp @@ -21,20 +21,11 @@ namespace storage { ProcessorCounters kGetDstBySrcCounters; void GetDstBySrcProcessor::process(const cpp2::GetDstBySrcRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/query/GetNeighborsProcessor.cpp b/src/storage/query/GetNeighborsProcessor.cpp index 9e8b5dd30ff..07c68d6a62a 100644 --- a/src/storage/query/GetNeighborsProcessor.cpp +++ b/src/storage/query/GetNeighborsProcessor.cpp @@ -21,20 +21,11 @@ namespace storage { ProcessorCounters kGetNeighborsCounters; void GetNeighborsProcessor::process(const cpp2::GetNeighborsRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/query/GetPropProcessor.cpp b/src/storage/query/GetPropProcessor.cpp index 21f7bda3d3d..81dfb5ef7e2 100644 --- a/src/storage/query/GetPropProcessor.cpp +++ b/src/storage/query/GetPropProcessor.cpp @@ -14,20 +14,11 @@ namespace storage { ProcessorCounters kGetPropCounters; void GetPropProcessor::process(const cpp2::GetPropRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/query/ScanEdgeProcessor.cpp b/src/storage/query/ScanEdgeProcessor.cpp index 717b1924662..c49d72906f5 100644 --- a/src/storage/query/ScanEdgeProcessor.cpp +++ b/src/storage/query/ScanEdgeProcessor.cpp @@ -16,20 +16,11 @@ namespace storage { ProcessorCounters kScanEdgeCounters; void ScanEdgeProcessor::process(const cpp2::ScanEdgeRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } diff --git a/src/storage/query/ScanVertexProcessor.cpp b/src/storage/query/ScanVertexProcessor.cpp index 429f514bdb5..6465b11de4b 100644 --- a/src/storage/query/ScanVertexProcessor.cpp +++ b/src/storage/query/ScanVertexProcessor.cpp @@ -18,20 +18,11 @@ namespace storage { ProcessorCounters kScanVertexCounters; void ScanVertexProcessor::process(const cpp2::ScanVertexRequest& req) { - try { - if (executor_ != nullptr) { - executor_->add([req, this]() { this->doProcess(req); }); - } else { - doProcess(req); - } - } catch (std::bad_alloc& e) { - memoryExceeded_ = true; - onError(); - } catch (std::exception& e) { - LOG(ERROR) << e.what(); - onError(); - } catch (...) { - onError(); + if (executor_ != nullptr) { + executor_->add( + [this, req]() { MemoryCheckScope wrapper(this, [this, req] { this->doProcess(req); }); }); + } else { + doProcess(req); } } From 28d070fb364c762fe5236f1b6c92acd1ded57118 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:44:56 +0800 Subject: [PATCH 16/51] fix bug (#5214) * fix bug * fix bug Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> --- src/kvstore/listener/elasticsearch/ESListener.cpp | 6 ++++++ tests/Makefile | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/kvstore/listener/elasticsearch/ESListener.cpp b/src/kvstore/listener/elasticsearch/ESListener.cpp index cdc4fe5205d..66927c8c5da 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.cpp +++ b/src/kvstore/listener/elasticsearch/ESListener.cpp @@ -95,6 +95,9 @@ void ESListener::pickTagAndEdgeData(BatchLogType type, } auto field = index.second.get_fields().front(); auto v = reader->getValueByName(field); + if (v.type() == Value::Type::NULLVALUE) { + continue; + } if (v.type() != Value::Type::STRING) { LOG(ERROR) << "Can't create fulltext index on type " << v.type(); } @@ -125,6 +128,9 @@ void ESListener::pickTagAndEdgeData(BatchLogType type, } auto field = index.second.get_fields().front(); auto v = reader->getValueByName(field); + if (v.type() == Value::Type::NULLVALUE) { + continue; + } if (v.type() != Value::Type::STRING) { LOG(ERROR) << "Can't create fulltext index on type " << v.type(); } diff --git a/tests/Makefile b/tests/Makefile index 5200dc86219..8e33ae1efc8 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -15,7 +15,7 @@ BUILD_DIR ?= $(CURR_DIR)/../build DEBUG ?= true J ?= 10 ENABLE_FT_INDEX ?= false -ES_ADDRESS ?= "locahost:9200" +ES_ADDRESS ?= 127.0.0.1:9200 ENABLE_SSL ?= false ENABLE_GRAPH_SSL ?= false ENABLE_META_SSL ?= false From 5de3449a04f77021edfb1c3e06946e5f685b6701 Mon Sep 17 00:00:00 2001 From: Alex Xing <90179377+SuperYoko@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:23:27 +0800 Subject: [PATCH 17/51] handle rpc error task status (#5212) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- .../processors/job/StorageJobExecutor.cpp | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/meta/processors/job/StorageJobExecutor.cpp b/src/meta/processors/job/StorageJobExecutor.cpp index 122f3e8b9c1..1934c7aedec 100644 --- a/src/meta/processors/job/StorageJobExecutor.cpp +++ b/src/meta/processors/job/StorageJobExecutor.cpp @@ -162,6 +162,7 @@ folly::Future StorageJobExecutor::execute() { // write all tasks first. std::vector data; + std::vector allTasks; for (auto i = 0U; i != addresses.size(); ++i) { TaskDescription task(space_, jobId_, i, addresses[i].first); auto taskKey = MetaKeyUtils::taskKey(task.getSpace(), task.getJobId(), task.getTaskId()); @@ -170,6 +171,7 @@ folly::Future StorageJobExecutor::execute() { task.getStartTime(), task.getStopTime(), task.getErrorCode()); + allTasks.emplace_back(std::move(task)); data.emplace_back(std::move(taskKey), std::move(taskVal)); } @@ -187,24 +189,59 @@ folly::Future StorageJobExecutor::execute() { } std::vector> futures; + futures.reserve(addresses.size()); for (auto& address : addresses) { // Will convert StorageAddr to AdminAddr in AdminClient futures.emplace_back(executeInternal(std::move(address.first), std::move(address.second))); } auto tries = folly::collectAll(std::move(futures)).get(); - for (auto& t : tries) { - if (t.hasException()) { - LOG(INFO) << t.exception().what(); + std::vector failedTasks; + for (size_t i = 0; i < tries.size(); i++) { + auto getFaildTask = [&](size_t taskId, nebula::cpp2::ErrorCode ec) { + auto task = allTasks[taskId]; + task.setStatus(cpp2::JobStatus::FAILED); + task.setErrorCode(ec); + return task; + }; + // taks id have the same index with address and futures. + if (tries[i].hasException()) { + LOG(INFO) << tries[i].exception().what(); rc = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + failedTasks.emplace_back(getFaildTask(i, rc)); continue; } - if (!t.value().ok()) { - LOG(INFO) << t.value().toString(); + if (!tries[i].value().ok()) { + LOG(INFO) << tries[i].value().toString(); rc = nebula::cpp2::ErrorCode::E_RPC_FAILURE; + failedTasks.emplace_back(getFaildTask(i, rc)); continue; } } + + if (!failedTasks.empty()) { + // write all tasks first. + std::vector faildKV; + for (auto task : failedTasks) { + auto taskKey = MetaKeyUtils::taskKey(task.getSpace(), task.getJobId(), task.getTaskId()); + auto taskVal = MetaKeyUtils::taskVal(task.getHost(), + task.getStatus(), + task.getStartTime(), + task.getStopTime(), + task.getErrorCode()); + faildKV.emplace_back(std::move(taskKey), std::move(taskVal)); + } + + kvstore_->asyncMultiPut( + kDefaultSpaceId, kDefaultPartId, std::move(faildKV), [&](nebula::cpp2::ErrorCode code) { + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(INFO) << "Some task is failed, but failed to set task status due to kv error:" + << apache::thrift::util::enumNameSafe(code); + } + baton.post(); + }); + baton.wait(); + } return rc; } From d4c97c220e532155e1e570cf1995f22eceb30465 Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Fri, 6 Jan 2023 17:26:15 +0800 Subject: [PATCH 18/51] chore: community badges refined (#5202) * chore: community badges refined * Update README-CN.md * Update README-CN.md * Update README-CN.md remove sifou and zhihu as aligned with the team * update linkedin URL Co-authored-by: Yee <2520865+yixinglu@users.noreply.github.com> --- README-CN.md | 22 +++++++++++----------- README.md | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/README-CN.md b/README-CN.md index fc2d5231815..7a2fcd7fc3b 100644 --- a/README-CN.md +++ b/README-CN.md @@ -8,21 +8,16 @@ WeiXin - - Zhihu - - - SegmentFault - Sina Weibo - nebula star + GitHub stars - nebula fork + GitHub forks +

# NebulaGraph 是什么？ @@ -108,13 +103,18 @@ NebulaGraph内核 1.x 与 2.x 数据格式、通信协议、客户端等均双 * 访问[官网](http://nebula-graph.com.cn/) * [![WeiXin](https://img.shields.io/badge/WeChat-%E5%BE%AE%E4%BF%A1-brightgreen)](https://user-images.githubusercontent.com/38887077/67449282-4362b300-f64c-11e9-878f-7efc373e5e55.jpg) * [![Sina Weibo](https://img.shields.io/badge/Weibo-%E5%BE%AE%E5%8D%9A-red)](https://weibo.com/p/1006067122684542/home?from=page_100606&mod=TAB#place) -* [知乎](https://www.zhihu.com/org/nebulagraph/activities) -* [SegmentFault](https://segmentfault.com/t/nebula) * Email: info@vesoft.com ## 加入 NebulaGraph 社区 -[![Discussions](https://img.shields.io/badge/GitHub_Discussion-000000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/vesoft-inc/nebula/discussions) [![Discourse](https://img.shields.io/badge/中文论坛-4285F4?style=for-the-badge&logo=discourse&logoColor=white)](https://discuss.nebula-graph.com.cn/) [![Slack](https://img.shields.io/badge/Slack-9F2B68?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) [![Tencent_Meeting](https://img.shields.io/badge/腾讯会议-2D8CFF?style=for-the-badge&logo=googlemeet&logoColor=white)](https://meeting.tencent.com/dm/F8NX1aRZ8PQv) [![Google Calendar](https://img.shields.io/badge/Calander-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://calendar.google.com/calendar/u/0?cid=Z29mbGttamM3ZTVlZ2hpazI2cmNlNXVnZThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) [![Meetup](https://img.shields.io/badge/Meetup-FF0000?style=for-the-badge&logo=meetup&logoColor=white)](https://www.meetup.com/nebulagraph/events/287180186?utm_medium=referral&utm_campaign=share-btn_savedevents_share_modal&utm_source=link) [![Meeting Archive](https://img.shields.io/badge/Community_wiki-808080?style=for-the-badge&logo=readthedocs&logoColor=white)](https://github.com/vesoft-inc/nebula-community/wiki) + + +| 加入 NebulaGraph 社区 | 加入方式 | +| ----------------------- | ------------------------------------------------------------ | +| 微信群 | [![WeChat Group](https://img.shields.io/badge/微信群-000000?style=for-the-badge&logo=wechat)](https://wj.qq.com/s2/8321168/8e2f/) | +| 提问 | [![Discourse](https://img.shields.io/badge/中文论坛-4285F4?style=for-the-badge&logo=discourse&logoColor=white)](https://discuss.nebula-graph.com.cn/) [![Stack Overflow](https://img.shields.io/badge/Stack%20Overflow-nebula--graph-orange?style=for-the-badge&logo=stack-overflow&logoColor=white)](https://stackoverflow.com/questions/tagged/nebula-graph) [![Discussions](https://img.shields.io/badge/GitHub_Discussion-000000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/vesoft-inc/nebula/discussions) | +| 聊天 | [![Chat History](https://img.shields.io/badge/Community%20Chat-000000?style=for-the-badge&logo=discord&logoColor=white)](https://community-chat.nebula-graph.io/) [![Slack](https://img.shields.io/badge/Slack-9F2B68?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) | +| NebulaGraph Meetup 活动 | [![Tencent_Meeting](https://img.shields.io/badge/腾讯会议-2D8CFF?style=for-the-badge&logo=googlemeet&logoColor=white)](https://meeting.tencent.com/dm/F8NX1aRZ8PQv) [![Google Calendar](https://img.shields.io/badge/Calander-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://calendar.google.com/calendar/u/0?cid=Z29mbGttamM3ZTVlZ2hpazI2cmNlNXVnZThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) [![Zoom](https://img.shields.io/badge/Zoom-2D8CFF?style=for-the-badge&logo=zoom&logoColor=white)](https://us02web.zoom.us/meeting/register/tZ0rcuypqDMvGdLuIm4VprTlx96wrEf062SH) [![Meetup](https://img.shields.io/badge/Meetup-FF0000?style=for-the-badge&logo=meetup&logoColor=white)](https://www.meetup.com/nebulagraph/events/) [![Meeting Archive](https://img.shields.io/badge/Meeting_Archive-808080?style=for-the-badge&logo=readthedocs&logoColor=white)](https://github.com/vesoft-inc/nebula-community/wiki) |
diff --git a/README.md b/README.md index 2ef22e51cb3..c525b32996a 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@
A distributed, scalable, lightning-fast graph database

+ + Stack Overflow + code coverage @@ -107,15 +110,21 @@ You can also freely deploy **NebulaGraph** as a back-end service to support your ## Contact +* [Community Chat](https://community-chat.nebula-graph.io/) * [Slack Channel](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) -* [Stack Overflow](https://stackoverflow.com/questions/tagged/nebulagraph) +* [Stack Overflow](https://stackoverflow.com/questions/tagged/nebula-graph) * Twitter: [@NebulaGraph](https://twitter.com/NebulaGraph) -* [LinkedIn Page](https://www.linkedin.com/company/vesoft-nebula-graph) +* [LinkedIn Page](https://www.linkedin.com/company/nebula-graph/) * Email: info@vesoft.com ## Community -[![Discussions](https://img.shields.io/badge/GitHub_Discussion-000000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/vesoft-inc/nebula/discussions) [![Slack](https://img.shields.io/badge/Slack-9F2B68?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) [![Zoom](https://img.shields.io/badge/Zoom-2D8CFF?style=for-the-badge&logo=zoom&logoColor=white)](https://us02web.zoom.us/meeting/register/tZ0rcuypqDMvGdLuIm4VprTlx96wrEf062SH) [![Google Calendar](https://img.shields.io/badge/Calander-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://calendar.google.com/calendar/u/0?cid=Z29mbGttamM3ZTVlZ2hpazI2cmNlNXVnZThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) [![Meetup](https://img.shields.io/badge/Meetup-FF0000?style=for-the-badge&logo=meetup&logoColor=white)](https://www.meetup.com/nebulagraph/events/287180186?utm_medium=referral&utm_campaign=share-btn_savedevents_share_modal&utm_source=link) [![Meeting Archive](https://img.shields.io/badge/Community_wiki-808080?style=for-the-badge&logo=readthedocs&logoColor=white)](https://github.com/vesoft-inc/nebula-community/wiki) [![Discourse](https://img.shields.io/badge/中文论坛-4285F4?style=for-the-badge&logo=discourse&logoColor=white)](https://discuss.nebula-graph.com.cn/) [![Tencent_Meeting](https://img.shields.io/badge/腾讯会议-2D8CFF?style=for-the-badge&logo=googlemeet&logoColor=white)](https://meeting.tencent.com/dm/F8NX1aRZ8PQv) +| Join NebulaGraph Community | Where to Find us | +| ----------------------------------- | ------------------------------------------------------------ | +| Asking Questions | [![Stack Overflow](https://img.shields.io/badge/Stack%20Overflow-nebula--graph-orange?style=for-the-badge&logo=stack-overflow&logoColor=white)](https://stackoverflow.com/questions/tagged/nebula-graph) [![Discussions](https://img.shields.io/badge/GitHub_Discussion-000000?style=for-the-badge&logo=github&logoColor=white)](https://github.com/vesoft-inc/nebula/discussions) | +| Chat with Community Members | [![Chat History](https://img.shields.io/badge/Community%20Chat-000000?style=for-the-badge&logo=discord&logoColor=white)](https://community-chat.nebula-graph.io/) [![Slack](https://img.shields.io/badge/Slack-9F2B68?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/nebulagraph/shared_invite/zt-7ybejuqa-NCZBroh~PCh66d9kOQj45g) | +| NebulaGraph Meetup | [![Google Calendar](https://img.shields.io/badge/Calander-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://calendar.google.com/calendar/u/0?cid=Z29mbGttamM3ZTVlZ2hpazI2cmNlNXVnZThAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) [![Zoom](https://img.shields.io/badge/Zoom-2D8CFF?style=for-the-badge&logo=zoom&logoColor=white)](https://us02web.zoom.us/meeting/register/tZ0rcuypqDMvGdLuIm4VprTlx96wrEf062SH) [![Meetup](https://img.shields.io/badge/Meetup-FF0000?style=for-the-badge&logo=meetup&logoColor=white)](https://www.meetup.com/nebulagraph/events/) [![Meeting Archive](https://img.shields.io/badge/Meeting_Archive-808080?style=for-the-badge&logo=readthedocs&logoColor=white)](https://github.com/vesoft-inc/nebula-community/wiki) | +| Chat, Asking, or Meeting in Chinese | [![WeChat Group](https://img.shields.io/badge/WeChat_Group-000000?style=for-the-badge&logo=wechat)](https://wj.qq.com/s2/8321168/8e2f/) [![Tencent_Meeting](https://img.shields.io/badge/腾讯会议-2D8CFF?style=for-the-badge&logo=googlemeet&logoColor=white)](https://meeting.tencent.com/dm/F8NX1aRZ8PQv) [![Discourse](https://img.shields.io/badge/中文论坛-4285F4?style=for-the-badge&logo=discourse&logoColor=white)](https://discuss.nebula-graph.com.cn/) |
From a068062411817358f4666223799e027e267cf378 Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Fri, 6 Jan 2023 18:26:34 +0800 Subject: [PATCH 19/51] Fix extend whtie space char. (#5213) * Fix extend whtie space char. * Format. Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/parser/scanner.lex | 13 ++++++++----- src/parser/test/ParserTest.cpp | 14 ++++++++++++++ src/parser/test/ScannerTest.cpp | 2 +- tests/tck/features/basic/Parser.feature | 16 ++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 tests/tck/features/basic/Parser.feature diff --git a/src/parser/scanner.lex b/src/parser/scanner.lex index a6fedde4a68..cbc40b42b17 100644 --- a/src/parser/scanner.lex +++ b/src/parser/scanner.lex @@ -35,7 +35,8 @@ static constexpr size_t MAX_STRING = 4096; %x LB_STR %x COMMENT -blank_without_newline ([ \t\r\xa0]) +nbsp (\xc2\xa0) +blank_without_newline ([ \t\r]|{nbsp}) blank ({blank_without_newline}|[\n]) blanks ({blank}+) @@ -57,17 +58,19 @@ HEX ([0-9a-fA-F]) OCT ([0-7]) IP_OCTET ([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]) -U [\x80-\xbf] -U2 [\xc2-\xdf] +U [\x80-\x9f\xa1-\xbf] +UA0 \xa0 +U2 [\xc3-\xdf] +UC2 \xc2 U3 [\xe0-\xee] U4 [\xf0-\xf4] -CHINESE {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U} +CHINESE {U2}{UA0}|{UC2}{U}|{U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U} CN_EN {CHINESE}|[a-zA-Z] CN_EN_NUM {CHINESE}|[_a-zA-Z0-9] LABEL {CN_EN}{CN_EN_NUM}* U3_FULL_WIDTH [\xe0-\xef] -CHINESE_FULL_WIDTH {U2}{U}|{U3_FULL_WIDTH}{U}{U}|{U4}{U}{U}{U} +CHINESE_FULL_WIDTH {U2}{UA0}|{UC2}{U}|{U2}{U}|{U3_FULL_WIDTH}{U}{U}|{U4}{U}{U}{U} CN_EN_FULL_WIDTH {CHINESE_FULL_WIDTH}|[a-zA-Z] CN_EN_NUM_FULL_WIDTH {CHINESE_FULL_WIDTH}|[_a-zA-Z0-9 ] LABEL_FULL_WIDTH {CN_EN_FULL_WIDTH}{CN_EN_NUM_FULL_WIDTH}* diff --git a/src/parser/test/ParserTest.cpp b/src/parser/test/ParserTest.cpp index 0d25a607c65..9bc976cc1a0 100644 --- a/src/parser/test/ParserTest.cpp +++ b/src/parser/test/ParserTest.cpp @@ -3350,4 +3350,18 @@ TEST_F(ParserTest, TestShowSentenceWithPipe) { ASSERT_TRUE(result.ok()) << result.status(); } } + +TEST_F(ParserTest, TestSpecialWhiteSpaceChar) { + { + std::string query = "SHOW\xC2\xA0SPACES"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } + { + std::string query = "SHOW \xC2\xA0SPACES\xC2\xA0"; + auto result = parse(query); + ASSERT_TRUE(result.ok()) << result.status(); + } +} + } // namespace nebula diff --git a/src/parser/test/ScannerTest.cpp b/src/parser/test/ScannerTest.cpp index f1b0675166e..6e6c4a32e51 100644 --- a/src/parser/test/ScannerTest.cpp +++ b/src/parser/test/ScannerTest.cpp @@ -533,7 +533,7 @@ TEST(Scanner, Basic) { CHECK_SEMANTIC_VALUE("label", TokenType::LABEL, "label"), CHECK_SEMANTIC_VALUE("label123", TokenType::LABEL, "label123"), // \xA0 is white space in UTF-8 too - CHECK_SEMANTIC_VALUE("\xA0" + CHECK_SEMANTIC_VALUE("\xC2\xA0" "abc", TokenType::LABEL, "abc"), diff --git a/tests/tck/features/basic/Parser.feature b/tests/tck/features/basic/Parser.feature new file mode 100644 index 00000000000..021e2ffa475 --- /dev/null +++ b/tests/tck/features/basic/Parser.feature @@ -0,0 +1,16 @@ +# Copyright (c) 2022 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: Parser + + Scenario: Test special white space character + When executing query: + """ + SHOW  SPACES + """ + Then the execution should be successful + When executing query: + """ + RETURN  1 + """ + Then the execution should be successful From fdbad534066b408e1ea836bca21df7b16d032817 Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Mon, 9 Jan 2023 11:14:33 +0800 Subject: [PATCH 20/51] Add lack tests of no role user. (#5196) Co-authored-by: Yee <2520865+yixinglu@users.noreply.github.com> --- tests/admin/test_permission.py | 14 ++++++++++++++ tests/common/utils.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/admin/test_permission.py b/tests/admin/test_permission.py index a1cf31fc54f..dda32fac188 100644 --- a/tests/admin/test_permission.py +++ b/tests/admin/test_permission.py @@ -891,6 +891,8 @@ def test_show_test(self): assert ret ret, self.guestClient = self.spawn_nebula_client_and_auth('guest', 'guest') assert ret + ret, self.nopClient = self.spawn_nebula_client_and_auth('nop', 'nop') + assert ret query = 'CREATE SPACE space4(partition_num=1, replica_factor=1, vid_type=FIXED_STRING(8))' resp = self.execute(query) @@ -930,6 +932,12 @@ def test_show_test(self): self.check_column_names(resp, expected_column_names) self.check_out_of_order_result(resp, expected_result) + resp = self.nopClient.execute(query) + self.check_resp_succeeded(resp) + expected_result = [] + self.check_column_names(resp, expected_column_names) + self.check_out_of_order_result(resp, expected_result) + query = 'SHOW ROLES IN space1' resp = self.guestClient.execute(query) self.check_resp_failed(resp, ttypes.ErrorCode.E_BAD_PERMISSION) @@ -987,6 +995,8 @@ def test_show_roles(self): assert ret ret, self.guestClient = self.spawn_nebula_client_and_auth('guest', 'guest') assert ret + ret, self.nopClient = self.spawn_nebula_client_and_auth('nop', 'nop') + assert ret query = 'SHOW ROLES IN space5' expected_result = [] @@ -1021,6 +1031,10 @@ def test_show_roles(self): self.check_resp_succeeded(resp) self.check_out_of_order_result(resp, expected_result) + expected_result = [] + resp = self.nopClient.execute(query) + self.check_resp_failed(resp, ttypes.ErrorCode.E_BAD_PERMISSION) + # clean up query = 'DROP SPACE IF EXISTS space5' resp = self.execute(query) diff --git a/tests/common/utils.py b/tests/common/utils.py index 760b638d146..77e7a34df07 100644 --- a/tests/common/utils.py +++ b/tests/common/utils.py @@ -442,7 +442,7 @@ def load_csv_data( def get_conn_pool(host: str, port: int, ssl_config: SSL_config): config = Config() - config.max_connection_pool_size = 20 + config.max_connection_pool_size = 30 config.timeout = 180000 # init connection pool pool = ConnectionPool() From b27b4e3b8cb8b60cd3ef7e8e2b29b6fa02bcc4af Mon Sep 17 00:00:00 2001 From: codesigner Date: Mon, 9 Jan 2023 18:23:06 +0800 Subject: [PATCH 21/51] remove memtracker DLOG (#5224) --- src/common/memory/MemoryTracker.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/memory/MemoryTracker.cpp b/src/common/memory/MemoryTracker.cpp index 548e4aacb38..a22a0aabf0d 100644 --- a/src/common/memory/MemoryTracker.cpp +++ b/src/common/memory/MemoryTracker.cpp @@ -15,7 +15,6 @@ ThreadMemoryStats::~ThreadMemoryStats() { // Return to global any reserved bytes on destruction if (reserved != 0) { MemoryStats::instance().freeGlobal(reserved); - DLOG(INFO) << std::this_thread::get_id() << " return reserved " << reserved; } } From 57ec8326278b74f4590454bdd53b632eb1608559 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Tue, 10 Jan 2023 09:58:21 +0800 Subject: [PATCH 22/51] Add tck cases for DDL (#5220) --- tests/tck/features/ddl/Ddl.feature | 298 +++++++++++++++++++++++++++++ tests/tck/utils/table.py | 2 + 2 files changed, 300 insertions(+) create mode 100644 tests/tck/features/ddl/Ddl.feature diff --git a/tests/tck/features/ddl/Ddl.feature b/tests/tck/features/ddl/Ddl.feature new file mode 100644 index 00000000000..bb9eb495fa8 --- /dev/null +++ b/tests/tck/features/ddl/Ddl.feature @@ -0,0 +1,298 @@ +# Copyright (c) 2023 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: DDL test + + Background: + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(30) | + | charset | utf8 | + | collate | utf8_bin | + + Scenario: Tag DDL + When executing query: + """ + CREATE TAG A(); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG IF NOT EXISTS A(id int, name string); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG B( + id int NOT NULL DEFAULT 0+0 COMMENT "primary key", + name string NOT NULL, + createDate DATETIME, location geography(polygon), + isVisited bool COMMENT "kHop search flag", + nickName TIME DEFAULT time() + ) + TTL_DURATION = 100, TTL_COL = "id", COMMENT = "TAG B"; + """ + Then the execution should be successful + When executing query: + """ + DESC TAG A; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + When executing query: + """ + DESC TAG B; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "id" | "int64" | "NO" | 0 | "primary key" | + | "name" | "string" | "NO" | | | + | "createDate" | "datetime" | "YES" | | | + | "location" | "geography(polygon)" | "YES" | | | + | "isVisited" | "bool" | "YES" | | "kHop search flag" | + | "nickName" | "time" | "YES" | "time()" | | + When executing query: + """ + ALTER TAG B DROP (name) + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_A_1 on A(); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_A_2 on A(id); + """ + Then a ExecutionError should be raised at runtime: Key not existed! + When executing query: + """ + CREATE TAG INDEX IF NOT EXISTS idx_A_3 on A(); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_1 on B(isVisited, id, nickName, name(1), createDate); + """ + Then a ExecutionError should be raised at runtime: Key not existed! + When executing query: + """ + ALTER TAG B ADD (name string) + """ + # IMHO, this is really confusing. https://github.com/vesoft-inc/nebula/issues/2671 + Then a ExecutionError should be raised at runtime: Existed! + When executing query: + """ + ALTER TAG B ADD (namex string) + """ + Then the execution should be successful + When executing query: + """ + ALTER TAG B CHANGE (isVisited bool) + """ + Then the execution should be successful + When executing query: + """ + ALTER TAG B CHANGE (isVisited int) + """ + Then a ExecutionError should be raised at runtime: Unsupported! + When executing query: + """ + CREATE TAG INDEX idx_B_2 on B(id); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_4 on B(namex); + """ + Then a ExecutionError should be raised at runtime: Invalid param! + When executing query: + """ + DROP TAG INDEX IF EXISTS idx_B_4; + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_5 on B(createDate); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_6 on B(location); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_7 on B(isVisited); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_8 on B(nickName); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_9 on B(id, nickName, namex(1), createDate); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_B_10 on B(id, nickName, namex(1)); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_E2_11 on E2(id, nickName, namex(1)); + """ + Then a ExecutionError should be raised at runtime: TagNotFound: Tag not existed! + When executing query: + """ + CREATE TAG INDEX idx_B_1 on B(isVisited, id, nickName, namex(1), createDate); + """ + Then the execution should be successful + + Scenario: Edge DDL + When executing query: + """ + CREATE EDGE E1(); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE IF NOT EXISTS E1(id int, name string); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE E2( + id int NOT NULL DEFAULT 0+0 COMMENT "primary key", + name string NOT NULL, + createDate DATETIME, location geography(polygon), + isVisited bool COMMENT "kHop search flag", + nickName TIME DEFAULT time() + ) + TTL_DURATION = 100, TTL_COL = "id", COMMENT = "EDGE E2"; + """ + Then the execution should be successful + When executing query: + """ + DESC EDGE E1; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + When executing query: + """ + DESC EDGE E2; + """ + Then the result should be, in any order: + | Field | Type | Null | Default | Comment | + | "id" | "int64" | "NO" | 0 | "primary key" | + | "name" | "string" | "NO" | | | + | "createDate" | "datetime" | "YES" | | | + | "location" | "geography(polygon)" | "YES" | | | + | "isVisited" | "bool" | "YES" | | "kHop search flag" | + | "nickName" | "time" | "YES" | "time()" | | + When executing query: + """ + ALTER EDGE E2 DROP (name) + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E1_1 on E1(); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E1_2 on E1(id); + """ + Then a ExecutionError should be raised at runtime: Key not existed! + When executing query: + """ + CREATE EDGE INDEX IF NOT EXISTS idx_E1_3 on E1(); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_1 on E2(isVisited, id, nickName, name(1), createDate); + """ + Then a ExecutionError should be raised at runtime: Key not existed! + When executing query: + """ + ALTER EDGE E2 ADD (name string) + """ + Then a ExecutionError should be raised at runtime: Existed! + When executing query: + """ + ALTER EDGE E2 ADD (namex string) + """ + Then the execution should be successful + When executing query: + """ + ALTER EDGE E2 CHANGE (isVisited bool) + """ + Then the execution should be successful + When executing query: + """ + ALTER EDGE E2 CHANGE (isVisited int) + """ + Then a ExecutionError should be raised at runtime: Unsupported! + When executing query: + """ + CREATE EDGE INDEX idx_E2_2 on E2(id); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_4 on E2(namex); + """ + Then a ExecutionError should be raised at runtime: Invalid param! + When executing query: + """ + DROP EDGE INDEX IF EXISTS idx_E2_4; + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_5 on E2(createDate); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_6 on E2(location); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_7 on E2(isVisited); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_8 on E2(nickName); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_9 on E2(id, nickName, namex(1), createDate); + """ + Then the execution should be successful + When executing query: + """ + CREATE EDGE INDEX idx_E2_10 on E2(id, nickName, namex(1)); + """ + Then the execution should be successful + When executing query: + """ + CREATE TAG INDEX idx_E2_11 on E2(id, nickName, namex(1)); + """ + Then a ExecutionError should be raised at runtime: TagNotFound: Tag not existed! + When executing query: + """ + CREATE EDGE INDEX idx_E2_1 on E2(isVisited, id, nickName, namex(1), createDate); + """ + Then the execution should be successful diff --git a/tests/tck/utils/table.py b/tests/tck/utils/table.py index 7505c2a30ff..ca3f550542c 100644 --- a/tests/tck/utils/table.py +++ b/tests/tck/utils/table.py @@ -13,6 +13,8 @@ def _parse_value(cell: str, variables: dict) -> Value: + if not cell: + cell = "EMPTY" m = pattern.match(cell) if m: var = m.group(1) From cc4c586f28194c22fdc3e87337183230eaf78759 Mon Sep 17 00:00:00 2001 From: Yee <2520865+yixinglu@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:05:33 +0800 Subject: [PATCH 23/51] more TCK tests for variable pattern match clause (#5215) * cleanup * same src/dst for variable length pattern * variable pattern in where clause * variable scope tests in path pattern * More tests * More tests Co-authored-by: jimingquan --- tests/Makefile | 2 +- tests/tck/features/match/Base.feature | 1 - .../match/VariableLengthPattern.feature | 140 ++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 8e33ae1efc8..caf4f48176c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -2,7 +2,7 @@ # # This source code is licensed under Apache 2.0 License. -.PHONY: fmt check check-and-diff init init-all clean test tck fail up down test-all ldbc +.PHONY: fmt check check-and-diff init init-all clean test tck fail up down test-all ldbc ps kill PYPI_MIRROR = https://mirrors.aliyun.com/pypi/simple/ # PYPI_MIRROR = http://pypi.mirrors.ustc.edu.cn/simple --trusted-host pypi.mirrors.ustc.edu.cn diff --git a/tests/tck/features/match/Base.feature b/tests/tck/features/match/Base.feature index a1f6f29c096..f8c0f8c07b1 100644 --- a/tests/tck/features/match/Base.feature +++ b/tests/tck/features/match/Base.feature @@ -1,7 +1,6 @@ # Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License. -@jie Feature: Basic match Background: diff --git a/tests/tck/features/match/VariableLengthPattern.feature b/tests/tck/features/match/VariableLengthPattern.feature index c3c19133787..a8e74e88ea5 100644 --- a/tests/tck/features/match/VariableLengthPattern.feature +++ b/tests/tck/features/match/VariableLengthPattern.feature @@ -375,3 +375,143 @@ Feature: Variable length Pattern match (m to n) """ Then the result should be, in any order: | v.player.name | + + Scenario: same src and dst for variable length pattern + When executing query: + """ + MATCH (v)-[e:like*0..0]-(v) + WHERE id(v) == 'Tim Duncan' + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 1 | + When executing query: + """ + MATCH (v)-[e:like*0..2]-(v) + WHERE id(v) == 'Tim Duncan' + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 5 | + When executing query: + """ + MATCH (v)-[e:like*2..3]-(v) + WHERE id(v) == 'Tim Duncan' + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 48 | + When executing query: + """ + MATCH (v)-[e:like*2..3]->(v) + WHERE id(v) == 'Tim Duncan' + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 4 | + When executing query: + """ + MATCH (v)-[e:like*0..]->(v) + WHERE id(v) == 'Tim Duncan' + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 13 | + + Scenario: variable length pattern and list expression + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*0..2]-(v2) + WHERE size([i in e WHERE i.likeness>90 | i])>1 + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 18 | + + @skip + # https://github.com/vesoft-inc/nebula/issues/5221 + Scenario: variable scope test in path pattern + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*0..2]-(v2) + WHERE size([i in e WHERE (v)-[i]-(v2) | i])>1 + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*0..2]-(v2)-[i]-(v3) + WHERE size([i in e WHERE (v)-[i]-(v2) | i])>1 + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*0..2]-(v2)-[i]-(v3) + WHERE size([i in e WHERE (v)-[i:like]-(v2) | i])>1 + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + When executing query: + """ + MATCH (v:player)-[e*2]->(n) + WHERE size([n in e WHERE (v{name:'Tim Duncan'})-[n]-()])>3 + RETURN v + """ + Then the result should be, in any order: + | v | + When executing query: + """ + MATCH (v:player)-[e*2]->()-[n]-() + WHERE size([n in e WHERE (v)-[n]-()])>0 + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + + Scenario: variable pattern in where clause + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e*0..2]-(v2) + WHERE NOT (v)-[:like*0..1]-(v2) + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 182 | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e*0..2]-(v2) + WHERE NOT (v)-[:like*1..2]-(v2) + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 182 | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*0..2]-(v2) + WHERE NOT (v)-[:like*0..1]-(v2) + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 56 | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*0..2]-(v2) + WHERE NOT (v)-[:like*1..2]-(v2) + RETURN count(*) AS cnt + """ + Then the result should be, in any order: + | cnt | + | 56 | From 34f4ae2903af5dd937c698b56c685688b9bedfaf Mon Sep 17 00:00:00 2001 From: Cheng Xuntao <7731943+xtcyclist@users.noreply.github.com> Date: Tue, 10 Jan 2023 15:37:49 +0800 Subject: [PATCH 24/51] Resumed the evaluation fo vertices in AttributeExpression (UTs included) (#5229) --- src/common/expression/AttributeExpression.cpp | 44 ++------------- .../test/AttributeExpressionTest.cpp | 54 ++++++------------- .../expression/test/ExpressionContextMock.h | 13 +---- .../test/ListComprehensionExpressionTest.cpp | 32 ++++------- .../test/PredicateExpressionTest.cpp | 34 ++++-------- .../features/expression/Attribute1.feature | 1 + .../expression/ListComprehension.feature | 6 +-- .../tck/features/expression/Predicate.feature | 8 +-- tests/tck/features/expression/Reduce.feature | 12 ++--- 9 files changed, 58 insertions(+), 146 deletions(-) diff --git a/src/common/expression/AttributeExpression.cpp b/src/common/expression/AttributeExpression.cpp index 026fce2bb0c..20438134d57 100644 --- a/src/common/expression/AttributeExpression.cpp +++ b/src/common/expression/AttributeExpression.cpp @@ -23,51 +23,17 @@ const Value &AttributeExpression::eval(ExpressionContext &ctx) { case Value::Type::MAP: return lvalue.getMap().at(rvalue.getStr()); case Value::Type::VERTEX: { - /* - * WARNING(Xuntao): Vertices shall not be evaluated as AttributeExpressions, - * since there shall always be a tag specified in the format of: - * var.tag.property. Due to design flaws, however, we have to keep - * this case segment. - */ if (rvalue.getStr() == kVid) { result_ = lvalue.getVertex().vid; return result_; } - /* - * WARNING(Xuntao): the following code snippet is dedicated to address the legacy - * problem of treating LabelTagProperty expressions as Attribute expressions. - * This snippet is necessary to allow users to write var.tag.prop in - * ListComprehensionExpression without making structural changes to the implementation. - */ - if (left()->kind() != Expression::Kind::kAttribute) { - if (right()->kind() == Expression::Kind::kConstant && - rvalue.type() == Value::Type::STRING) { - auto rStr = rvalue.getStr(); - for (auto &tag : lvalue.getVertex().tags) { - if (rStr.compare(tag.name) == 0) { - return lvalue; - } - } - LOG(ERROR) << "Tag not found for: " << rStr - << "Please check whether the related expression " - << "follows the format of vertex.tag.property."; - } - } else if (left()->kind() == Expression::Kind::kAttribute && - dynamic_cast(left())->right()->kind() == - Expression::Kind::kConstant) { - auto &tagName = dynamic_cast(left())->right()->eval(ctx).getStr(); - for (auto &tag : lvalue.getVertex().tags) { - if (tagName.compare(tag.name) != 0) { - continue; - } else { - auto iter = tag.props.find(rvalue.getStr()); - if (iter != tag.props.end()) { - return iter->second; - } - } + for (auto &tag : lvalue.getVertex().tags) { + auto iter = tag.props.find(rvalue.getStr()); + if (iter != tag.props.end()) { + return iter->second; } } - return Value::kNullUnknownProp; + return Value::kNullValue; } case Value::Type::EDGE: { DCHECK(!rvalue.getStr().empty()); diff --git a/src/common/expression/test/AttributeExpressionTest.cpp b/src/common/expression/test/AttributeExpressionTest.cpp index 615034cfe52..1258e953bfa 100644 --- a/src/common/expression/test/AttributeExpressionTest.cpp +++ b/src/common/expression/test/AttributeExpressionTest.cpp @@ -2,10 +2,7 @@ * * This source code is licensed under Apache 2.0 License. */ -#include - #include "common/expression/test/TestBase.h" -#include "graph/util/ExpressionUtils.h" namespace nebula { @@ -85,8 +82,6 @@ TEST_F(AttributeExpressionTest, VertexAttribute) { Vertex vertex; vertex.vid = "vid"; vertex.tags.resize(2); - vertex.tags[0].name = "tag0"; - vertex.tags[1].name = "tag1"; vertex.tags[0].props = { {"Venus", "Mars"}, {"Mull", "Kintyre"}, @@ -96,42 +91,27 @@ TEST_F(AttributeExpressionTest, VertexAttribute) { {"Tug", "War"}, {"Venus", "RocksShow"}, }; - std::unordered_map aliasTypeMap = {{"v", graph::AliasType::kNode}}; - ExpressionContextMock expContext; - expContext.setVar("v", Value(vertex)); - { - auto *left = LabelExpression::make(&pool, "v"); - auto *right = ConstantExpression::make(&pool, "tag0"); - auto *labelAttribute = LabelAttributeExpression::make(&pool, left, right); - auto expr = - AttributeExpression::make(&pool, labelAttribute, ConstantExpression::make(&pool, "Mull")); - auto rewriteExpr = - graph::ExpressionUtils::rewriteAttr2LabelTagProp(expr->clone(), aliasTypeMap); - auto value = Expression::eval(rewriteExpr, expContext); + { + auto *left = ConstantExpression::make(&pool, Value(vertex)); + auto *right = LabelExpression::make(&pool, "Mull"); + auto expr = AttributeExpression::make(&pool, left, right); + auto value = Expression::eval(expr, gExpCtxt); ASSERT_TRUE(value.isStr()); ASSERT_EQ("Kintyre", value.getStr()); } { - auto *left = LabelExpression::make(&pool, "v"); - auto *right = ConstantExpression::make(&pool, "tag1"); - auto *labelAttribute = LabelAttributeExpression::make(&pool, left, right); - auto expr = - AttributeExpression::make(&pool, labelAttribute, ConstantExpression::make(&pool, "Bip")); - auto rewriteExpr = - graph::ExpressionUtils::rewriteAttr2LabelTagProp(expr->clone(), aliasTypeMap); - auto value = Expression::eval(rewriteExpr, expContext); + auto *left = ConstantExpression::make(&pool, Value(vertex)); + auto *right = LabelExpression::make(&pool, "Bip"); + auto expr = AttributeExpression::make(&pool, left, right); + auto value = Expression::eval(expr, gExpCtxt); ASSERT_TRUE(value.isStr()); ASSERT_EQ("Bop", value.getStr()); } { - auto *left = LabelExpression::make(&pool, "v"); - auto *right = ConstantExpression::make(&pool, "tag0"); - auto *labelAttribute = LabelAttributeExpression::make(&pool, left, right); - auto expr = - AttributeExpression::make(&pool, labelAttribute, ConstantExpression::make(&pool, "Venus")); - auto rewriteExpr = - graph::ExpressionUtils::rewriteAttr2LabelTagProp(expr->clone(), aliasTypeMap); - auto value = Expression::eval(rewriteExpr, expContext); + auto *left = ConstantExpression::make(&pool, Value(vertex)); + auto *right = LabelExpression::make(&pool, "Venus"); + auto expr = AttributeExpression::make(&pool, left, right); + auto value = Expression::eval(expr, gExpCtxt); ASSERT_TRUE(value.isStr()); ASSERT_EQ("Mars", value.getStr()); } @@ -139,7 +119,7 @@ TEST_F(AttributeExpressionTest, VertexAttribute) { auto *left = ConstantExpression::make(&pool, Value(vertex)); auto *right = LabelExpression::make(&pool, "_vid"); auto expr = AttributeExpression::make(&pool, left, right); - auto value = Expression::eval(expr, expContext); + auto value = Expression::eval(expr, gExpCtxt); ASSERT_TRUE(value.isStr()); ASSERT_EQ("vid", value.getStr()); } @@ -154,7 +134,7 @@ TEST_F(AttributeExpressionTest, DateTimeAttribute) { auto *right = LabelExpression::make(&pool, "not exist attribute"); auto expr = AttributeExpression::make(&pool, left, right); auto value = Expression::eval(expr, gExpCtxt); - ASSERT_EQ(Value::kNullUnknownProp, value); + ASSERT_EQ(Value::kNullValue, value); } { auto *left = ConstantExpression::make(&pool, Value(dt)); @@ -168,7 +148,7 @@ TEST_F(AttributeExpressionTest, DateTimeAttribute) { auto *right = LabelExpression::make(&pool, "not exist attribute"); auto expr = AttributeExpression::make(&pool, left, right); auto value = Expression::eval(expr, gExpCtxt); - ASSERT_EQ(Value::kNullUnknownProp, value); + ASSERT_EQ(Value::kNullValue, value); } { auto *left = ConstantExpression::make(&pool, Value(d)); @@ -182,7 +162,7 @@ TEST_F(AttributeExpressionTest, DateTimeAttribute) { auto *right = LabelExpression::make(&pool, "not exist attribute"); auto expr = AttributeExpression::make(&pool, left, right); auto value = Expression::eval(expr, gExpCtxt); - ASSERT_EQ(Value::kNullUnknownProp, value); + ASSERT_EQ(Value::kNullValue, value); } { auto *left = ConstantExpression::make(&pool, Value(t)); diff --git a/src/common/expression/test/ExpressionContextMock.h b/src/common/expression/test/ExpressionContextMock.h index c280de58ac4..fab7d557414 100644 --- a/src/common/expression/test/ExpressionContextMock.h +++ b/src/common/expression/test/ExpressionContextMock.h @@ -24,16 +24,7 @@ class ExpressionContextMock final : public ExpressionContext { } void setInnerVar(const std::string& var, Value val) override { - if (var == "xxx") { - if (vals_.empty()) { - vals_.emplace_back(val); - indices_[var] = vals_.size() - 1; - } else { - vals_[indices_[var]] = val; - } - } else { - exprValueMap_[var] = std::move(val); - } + exprValueMap_[var] = std::move(val); } const Value& getInnerVar(const std::string& var) const override { @@ -152,7 +143,7 @@ class ExpressionContextMock final : public ExpressionContext { void setVar(const std::string& var, Value val) override { // used by tests of list comprehesion, predicate or reduce - if (var == "n" || var == "p" || var == "totalNum" || var == "v") { + if (var == "n" || var == "p" || var == "totalNum") { vals_.emplace_back(val); indices_[var] = vals_.size() - 1; } diff --git a/src/common/expression/test/ListComprehensionExpressionTest.cpp b/src/common/expression/test/ListComprehensionExpressionTest.cpp index 6f762cc9a29..6e5931c0fbd 100644 --- a/src/common/expression/test/ListComprehensionExpressionTest.cpp +++ b/src/common/expression/test/ListComprehensionExpressionTest.cpp @@ -37,7 +37,7 @@ TEST_F(ListComprehensionExpressionTest, ListComprehensionEvaluate) { ASSERT_EQ(expected, value.getList()); } { - // [n IN nodes(p) | n.player.age + 5] + // [n IN nodes(p) | n.age + 5] auto v1 = Vertex("101", {Tag("player", {{"name", "joe"}, {"age", 18}})}); auto v2 = Vertex("102", {Tag("player", {{"name", "amber"}, {"age", 19}})}); auto v3 = Vertex("103", {Tag("player", {{"name", "shawdan"}, {"age", 20}})}); @@ -46,25 +46,19 @@ TEST_F(ListComprehensionExpressionTest, ListComprehensionEvaluate) { path.steps.emplace_back(Step(v2, 1, "like", 0, {})); path.steps.emplace_back(Step(v3, 1, "like", 0, {})); gExpCtxt.setVar("p", path); - std::unordered_map aliasTypeMap = { - {"xxx", graph::AliasType::kNode}}; + ArgumentList *argList = ArgumentList::make(&pool); argList->addArgument(VariableExpression::make(&pool, "p")); auto expr = ListComprehensionExpression::make( &pool, - "xxx", + "n", FunctionCallExpression::make(&pool, "nodes", argList), nullptr, ArithmeticExpression::makeAdd( &pool, - graph::ExpressionUtils::rewriteAttr2LabelTagProp( - AttributeExpression::make( - &pool, - LabelAttributeExpression::make(&pool, - LabelExpression::make(&pool, "xxx"), - ConstantExpression::make(&pool, "player")), - ConstantExpression::make(&pool, "age")), - aliasTypeMap), + AttributeExpression::make(&pool, + VariableExpression::makeInner(&pool, "n"), + ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 5))); auto value = Expression::eval(expr, gExpCtxt); @@ -94,8 +88,6 @@ TEST_F(ListComprehensionExpressionTest, ListComprehensionExprToString) { { ArgumentList *argList = ArgumentList::make(&pool); argList->addArgument(LabelExpression::make(&pool, "p")); - std::unordered_map aliasTypeMap = { - {"n", graph::AliasType::kNode}}; auto expr = ListComprehensionExpression::make( &pool, "n", @@ -103,16 +95,10 @@ TEST_F(ListComprehensionExpressionTest, ListComprehensionExprToString) { nullptr, ArithmeticExpression::makeAdd( &pool, - graph::ExpressionUtils::rewriteAttr2LabelTagProp( - AttributeExpression::make( - &pool, - LabelAttributeExpression::make(&pool, - LabelExpression::make(&pool, "n"), - ConstantExpression::make(&pool, "player")), - ConstantExpression::make(&pool, "age")), - aliasTypeMap), + LabelAttributeExpression::make( + &pool, LabelExpression::make(&pool, "n"), ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 10))); - ASSERT_EQ("[n IN nodes(p) | (n.player.age+10)]", expr->toString()); + ASSERT_EQ("[n IN nodes(p) | (n.age+10)]", expr->toString()); } { auto listItems = ExpressionList::make(&pool); diff --git a/src/common/expression/test/PredicateExpressionTest.cpp b/src/common/expression/test/PredicateExpressionTest.cpp index 919533776c6..2fcc9c8f1b2 100644 --- a/src/common/expression/test/PredicateExpressionTest.cpp +++ b/src/common/expression/test/PredicateExpressionTest.cpp @@ -31,7 +31,7 @@ TEST_F(PredicateExpressionTest, PredicateEvaluate) { ASSERT_EQ(false, value.getBool()); } { - // any(xxx IN nodes(p) WHERE xxx.player.age >= 19) + // any(n IN nodes(p) WHERE n.age >= 19) auto v1 = Vertex("101", {Tag("player", {{"name", "joe"}, {"age", 18}})}); auto v2 = Vertex("102", {Tag("player", {{"name", "amber"}, {"age", 19}})}); auto v3 = Vertex("103", {Tag("player", {{"name", "shawdan"}, {"age", 20}})}); @@ -40,25 +40,19 @@ TEST_F(PredicateExpressionTest, PredicateEvaluate) { path.steps.emplace_back(Step(v2, 1, "like", 0, {})); path.steps.emplace_back(Step(v3, 1, "like", 0, {})); gExpCtxt.setVar("p", path); - std::unordered_map aliasTypeMap = { - {"xxx", graph::AliasType::kNode}}; + ArgumentList *argList = ArgumentList::make(&pool); argList->addArgument(VariableExpression::make(&pool, "p")); auto expr = PredicateExpression::make( &pool, "any", - "xxx", + "n", FunctionCallExpression::make(&pool, "nodes", argList), RelationalExpression::makeGE( &pool, - graph::ExpressionUtils::rewriteAttr2LabelTagProp( - AttributeExpression::make( - &pool, - LabelAttributeExpression::make(&pool, - LabelExpression::make(&pool, "xxx"), - ConstantExpression::make(&pool, "player")), - ConstantExpression::make(&pool, "age")), - aliasTypeMap), + AttributeExpression::make(&pool, + VariableExpression::makeInner(&pool, "n"), + ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 19))); auto value = Expression::eval(expr, gExpCtxt); @@ -96,25 +90,19 @@ TEST_F(PredicateExpressionTest, PredicateEvaluate) { path.steps.emplace_back(Step(v2, 1, "like", 0, {})); path.steps.emplace_back(Step(v3, 1, "like", 0, {})); gExpCtxt.setVar("p", path); - std::unordered_map aliasTypeMap = { - {"xxx", graph::AliasType::kNode}}; + ArgumentList *argList = ArgumentList::make(&pool); argList->addArgument(VariableExpression::make(&pool, "p")); auto expr = PredicateExpression::make( &pool, "none", - "xxx", + "n", FunctionCallExpression::make(&pool, "nodes", argList), RelationalExpression::makeGE( &pool, - graph::ExpressionUtils::rewriteAttr2LabelTagProp( - AttributeExpression::make( - &pool, - LabelAttributeExpression::make(&pool, - LabelExpression::make(&pool, "xxx"), - ConstantExpression::make(&pool, "player")), - ConstantExpression::make(&pool, "age")), - aliasTypeMap), + AttributeExpression::make(&pool, + VariableExpression::makeInner(&pool, "n"), + ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 19))); auto value = Expression::eval(expr, gExpCtxt); diff --git a/tests/tck/features/expression/Attribute1.feature b/tests/tck/features/expression/Attribute1.feature index 92cfcf9387a..d153258ee17 100644 --- a/tests/tck/features/expression/Attribute1.feature +++ b/tests/tck/features/expression/Attribute1.feature @@ -1,6 +1,7 @@ # Copyright (c) 2021 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License. +@skip Feature: Attribute using test Background: diff --git a/tests/tck/features/expression/ListComprehension.feature b/tests/tck/features/expression/ListComprehension.feature index a94bbd10e1d..bcaa62d42eb 100644 --- a/tests/tck/features/expression/ListComprehension.feature +++ b/tests/tck/features/expression/ListComprehension.feature @@ -53,8 +53,8 @@ Feature: ListComprehension When executing query: """ MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) - RETURN [n IN nodes(p) WHERE n.player.name - NOT STARTS WITH "LeBron" | n.player.age + 100] AS r + RETURN [n IN nodes(p) WHERE n.name + NOT STARTS WITH "LeBron" | n.age + 100] AS r """ Then the result should be, in any order: | r | @@ -67,7 +67,7 @@ Feature: ListComprehension When executing query: """ MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) - RETURN [n IN nodes(p) | n.player.age + 100] AS r + RETURN [n IN nodes(p) | n.age + 100] AS r """ Then the result should be, in any order: | r | diff --git a/tests/tck/features/expression/Predicate.feature b/tests/tck/features/expression/Predicate.feature index a059899f1d2..3853c812d10 100644 --- a/tests/tck/features/expression/Predicate.feature +++ b/tests/tck/features/expression/Predicate.feature @@ -285,9 +285,9 @@ Feature: Predicate """ MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) RETURN - nodes(p)[0].player.name AS n1, - nodes(p)[1].player.name AS n2, - all(n IN nodes(p) WHERE n.player.name NOT STARTS WITH "D") AS b + nodes(p)[0].name AS n1, + nodes(p)[1].name AS n2, + all(n IN nodes(p) WHERE n.name NOT STARTS WITH "D") AS b """ Then the result should be, in any order: | n1 | n2 | b | @@ -300,7 +300,7 @@ Feature: Predicate When executing query: """ MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) - RETURN single(n IN nodes(p) WHERE n.player.age > 40) AS b + RETURN single(n IN nodes(p) WHERE n.age > 40) AS b """ Then the result should be, in any order: | b | diff --git a/tests/tck/features/expression/Reduce.feature b/tests/tck/features/expression/Reduce.feature index 7026fb26d9a..cae899f1544 100644 --- a/tests/tck/features/expression/Reduce.feature +++ b/tests/tck/features/expression/Reduce.feature @@ -40,9 +40,9 @@ Feature: Reduce """ MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) RETURN - nodes(p)[0].player.age AS age1, - nodes(p)[1].player.age AS age2, - reduce(totalAge = 100, n IN nodes(p) | totalAge + n.player.age) AS r + nodes(p)[0].age AS age1, + nodes(p)[1].age AS age2, + reduce(totalAge = 100, n IN nodes(p) | totalAge + n.age) AS r """ Then the result should be, in any order: | age1 | age2 | r | @@ -55,9 +55,9 @@ Feature: Reduce When executing query: """ MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) - RETURN nodes(p)[0].player.age AS age1, - nodes(p)[1].player.age AS age2, - reduce(x = 10, n IN nodes(p) | n.player.age - x) AS x + RETURN nodes(p)[0].age AS age1, + nodes(p)[1].age AS age2, + reduce(x = 10, n IN nodes(p) | n.age - x) AS x """ Then the result should be, in any order: | age1 | age2 | x | From f41698a924603cf981824e0c58b73793676d8096 Mon Sep 17 00:00:00 2001 From: codesigner Date: Tue, 10 Jan 2023 16:53:36 +0800 Subject: [PATCH 25/51] add memtracker flags to conf (#5231) * add memtracker flags to conf * typo * refine * refine --- conf/nebula-graphd.conf.default | 16 ++++++++++++++++ conf/nebula-graphd.conf.production | 16 ++++++++++++++++ conf/nebula-storaged.conf.default | 16 ++++++++++++++++ conf/nebula-storaged.conf.production | 16 ++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default index 6f184327df6..871ddd667c3 100644 --- a/conf/nebula-graphd.conf.default +++ b/conf/nebula-graphd.conf.default @@ -97,3 +97,19 @@ ########## session ########## # Maximum number of sessions that can be created per IP and per user --max_sessions_per_ip_per_user=300 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=true +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 diff --git a/conf/nebula-graphd.conf.production b/conf/nebula-graphd.conf.production index d5b07642394..ad28a43cf39 100644 --- a/conf/nebula-graphd.conf.production +++ b/conf/nebula-graphd.conf.production @@ -96,3 +96,19 @@ ########## session ########## # Maximum number of sessions that can be created per IP and per user --max_sessions_per_ip_per_user=300 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=true +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default index 34c345ac9ed..4d102e616a8 100644 --- a/conf/nebula-storaged.conf.default +++ b/conf/nebula-storaged.conf.default @@ -122,3 +122,19 @@ --rebuild_index_part_rate_limit=4194304 # The amount of data sent in each batch when leader synchronizes rebuilding index --rebuild_index_batch_size=1048576 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=true +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production index d0004f06fc5..db71776e61f 100644 --- a/conf/nebula-storaged.conf.production +++ b/conf/nebula-storaged.conf.production @@ -123,3 +123,19 @@ --rebuild_index_part_rate_limit=4194304 # The amount of data sent in each batch when leader synchronizes rebuilding index --rebuild_index_batch_size=1048576 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=true +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 From 2f0ccaa76b2515e451a63ac44b645268d402fbf5 Mon Sep 17 00:00:00 2001 From: "pengwei.song" <90180021+pengweisong@users.noreply.github.com> Date: Tue, 10 Jan 2023 17:46:51 +0800 Subject: [PATCH 26/51] add balance job type to filter when create backup (#5228) * add more job type to filter when create backup * log add job * add log before acquire snapshot lock Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/meta/processors/admin/CreateBackupProcessor.cpp | 3 +++ src/meta/processors/job/JobManager.cpp | 8 +++++++- src/meta/test/CreateBackupProcessorTest.cpp | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/meta/processors/admin/CreateBackupProcessor.cpp b/src/meta/processors/admin/CreateBackupProcessor.cpp index 9d2e10b346a..5735e55ae66 100644 --- a/src/meta/processors/admin/CreateBackupProcessor.cpp +++ b/src/meta/processors/admin/CreateBackupProcessor.cpp @@ -85,9 +85,11 @@ void CreateBackupProcessor::process(const cpp2::CreateBackupReq& req) { // make sure there is no index job std::unordered_set jobTypes{cpp2::JobType::REBUILD_TAG_INDEX, cpp2::JobType::REBUILD_EDGE_INDEX, + cpp2::JobType::REBUILD_FULLTEXT_INDEX, cpp2::JobType::COMPACT, cpp2::JobType::INGEST, cpp2::JobType::DATA_BALANCE, + cpp2::JobType::ZONE_BALANCE, cpp2::JobType::LEADER_BALANCE}; auto result = jobMgr->checkTypeJobRunning(jobTypes); if (!nebula::ok(result)) { @@ -105,6 +107,7 @@ void CreateBackupProcessor::process(const cpp2::CreateBackupReq& req) { } folly::SharedMutex::WriteHolder holder(LockUtils::snapshotLock()); + LOG(INFO) << "Start to create checkpoints in all hosts."; // get active storage host list auto activeHostsRet = ActiveHostsMan::getActiveHosts(kvstore_); if (!nebula::ok(activeHostsRet)) { diff --git a/src/meta/processors/job/JobManager.cpp b/src/meta/processors/job/JobManager.cpp index c9d31c417fb..8b28bb0862a 100644 --- a/src/meta/processors/job/JobManager.cpp +++ b/src/meta/processors/job/JobManager.cpp @@ -567,8 +567,13 @@ nebula::cpp2::ErrorCode JobManager::addJob(JobDescription jobDesc) { if (rc == nebula::cpp2::ErrorCode::SUCCEEDED) { enqueue(spaceId, jobId, JbOp::ADD, jobDesc.getJobType()); inFlightJobs_.emplace(std::move(jobId), std::move(jobDesc)); + LOG(INFO) << folly::sformat("Add job successfully, job id={}, job type={}", + jobId, + apache::thrift::util::enumNameSafe(jobDesc.getJobType())); } else { - LOG(INFO) << "Add Job Failed"; + LOG(INFO) << folly::sformat("Add job failed, job id={}, job type={}", + jobId, + apache::thrift::util::enumNameSafe(jobDesc.getJobType())); if (rc != nebula::cpp2::ErrorCode::E_LEADER_CHANGED) { rc = nebula::cpp2::ErrorCode::E_ADD_JOB_FAILURE; } @@ -1002,6 +1007,7 @@ ErrorOr JobManager::checkTypeJobRunning( auto jobDesc = nebula::value(optJobRet); auto jType = jobDesc.getJobType(); if (jobTypes.find(jType) == jobTypes.end()) { + LOG(INFO) << "skip job type:" << apache::thrift::util::enumNameSafe(jType); continue; } diff --git a/src/meta/test/CreateBackupProcessorTest.cpp b/src/meta/test/CreateBackupProcessorTest.cpp index 4291f19c834..d0bb559b371 100644 --- a/src/meta/test/CreateBackupProcessorTest.cpp +++ b/src/meta/test/CreateBackupProcessorTest.cpp @@ -272,9 +272,11 @@ TEST_F(CreateBackupProcessorTest, Basic) { TEST_F(CreateBackupProcessorTest, RunningJobs) { std::vector jobTypes{cpp2::JobType::REBUILD_TAG_INDEX, cpp2::JobType::REBUILD_EDGE_INDEX, + cpp2::JobType::REBUILD_FULLTEXT_INDEX, cpp2::JobType::COMPACT, cpp2::JobType::INGEST, cpp2::JobType::DATA_BALANCE, + cpp2::JobType::ZONE_BALANCE, cpp2::JobType::LEADER_BALANCE}; JobID jobId = 1; for (auto jobType : jobTypes) { From 70e786444ba93420650d74233457c9065bc21f84 Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Wed, 11 Jan 2023 11:33:42 +0800 Subject: [PATCH 27/51] Fix update sessions when leader change happens (#5225) * Fix udpate sessions when leader change happens * Handle errors on the graph side * Address comments * Address comments --- src/graph/session/GraphSessionManager.cpp | 13 +++++++++--- .../session/SessionManagerProcessor.cpp | 20 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/graph/session/GraphSessionManager.cpp b/src/graph/session/GraphSessionManager.cpp index fce4206b088..e11fbc57d03 100644 --- a/src/graph/session/GraphSessionManager.cpp +++ b/src/graph/session/GraphSessionManager.cpp @@ -5,6 +5,7 @@ #include "graph/session/GraphSessionManager.h" #include "common/base/Base.h" +#include "common/base/Status.h" #include "common/stats/StatsManager.h" #include "common/time/WallClock.h" #include "graph/service/GraphFlags.h" @@ -258,8 +259,9 @@ void GraphSessionManager::updateSessionsToMeta() { auto handleKilledQueries = [this](auto&& resp) { if (!resp.ok()) { LOG(ERROR) << "Update sessions failed: " << resp.status(); - return Status::Error("Update sessions failed: %s", resp.status().toString().c_str()); + return; } + auto& killedQueriesForEachSession = *resp.value().killed_queries_ref(); for (auto& killedQueries : killedQueriesForEachSession) { auto sessionId = killedQueries.first; @@ -276,19 +278,24 @@ void GraphSessionManager::updateSessionsToMeta() { VLOG(1) << "Kill query, session: " << sessionId << " plan: " << epId; } } - return Status::OK(); }; // The response from meta contains sessions that are marked as killed, so we need to clean the // local cache and update statistics auto handleKilledSessions = [this](auto&& resp) { + if (!resp.ok()) { + LOG(ERROR) << "Update sessions failed: " << resp.status(); + return; + } + auto killSessions = resp.value().get_killed_sessions(); removeSessionFromLocalCache(killSessions); }; auto result = metaClient_->updateSessions(sessions).get(); if (!result.ok()) { - LOG(ERROR) << "Update sessions failed: " << result; + LOG(ERROR) << "Update sessions failed: " << result.status(); + return; } handleKilledQueries(result); handleKilledSessions(result); diff --git a/src/meta/processors/session/SessionManagerProcessor.cpp b/src/meta/processors/session/SessionManagerProcessor.cpp index 75a30578add..e33a5341b60 100644 --- a/src/meta/processors/session/SessionManagerProcessor.cpp +++ b/src/meta/processors/session/SessionManagerProcessor.cpp @@ -57,11 +57,15 @@ void UpdateSessionsProcessor::process(const cpp2::UpdateSessionsReq& req) { if (!nebula::ok(ret)) { auto errCode = nebula::error(ret); LOG(INFO) << "Session id '" << sessionId << "' not found"; - // If the session requested to be updated can not be found in meta, the session has been - // killed + // If the session requested to be updated can not be found in meta, we consider the session + // has been killed if (errCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { killedSessions.emplace_back(sessionId); continue; + } else { + handleErrorCode(errCode); + onFinished(); + return; } } @@ -169,10 +173,18 @@ void RemoveSessionProcessor::process(const cpp2::RemoveSessionReq& req) { auto sessionKey = MetaKeyUtils::sessionKey(sessionId); auto ret = doGet(sessionKey); - // If the session is not found, we should continue to remove other sessions. if (!nebula::ok(ret)) { + auto errCode = nebula::error(ret); LOG(INFO) << "Session id `" << sessionId << "' not found"; - continue; + + // If the session is not found, we should continue to remove other sessions. + if (errCode == nebula::cpp2::ErrorCode::E_KEY_NOT_FOUND) { + continue; + } else { // for other error like leader change, we handle the error and return. + handleErrorCode(errCode); + onFinished(); + return; + } } // Remove session key from kvstore From 9c9fb93260a679a5f413be824b69ba07ddf1d328 Mon Sep 17 00:00:00 2001 From: jimingquan Date: Wed, 11 Jan 2023 12:04:19 +0800 Subject: [PATCH 28/51] fix match step range (#5216) * use smart pointer change raw pointer * fix error * fix test error * address comment Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- .../expression/MatchPathPatternExpression.h | 4 + src/graph/context/ast/CypherAstContext.h | 2 +- src/graph/executor/algo/ShortestPathBase.h | 2 +- src/graph/executor/query/TraverseExecutor.cpp | 18 ++-- src/graph/executor/query/TraverseExecutor.h | 5 +- ...tEdgesTransformAppendVerticesLimitRule.cpp | 3 +- .../optimizer/rule/GetEdgesTransformRule.cpp | 3 +- src/graph/planner/match/MatchPathPlanner.cpp | 12 ++- .../planner/match/ShortestPathPlanner.cpp | 7 +- src/graph/planner/plan/Algo.cpp | 2 +- src/graph/planner/plan/Algo.h | 6 +- src/graph/planner/plan/Query.cpp | 2 +- src/graph/planner/plan/Query.h | 10 +-- src/graph/validator/MatchValidator.cpp | 65 +++++++------- src/graph/validator/MatchValidator.h | 3 +- src/parser/MatchPath.h | 2 +- tests/tck/features/match/PathExpr.feature | 85 +++++++++++++++++++ .../match/PathExprRefLocalVariable.feature | 15 ---- .../match/VariableLengthPattern.feature | 3 +- 19 files changed, 164 insertions(+), 85 deletions(-) diff --git a/src/common/expression/MatchPathPatternExpression.h b/src/common/expression/MatchPathPatternExpression.h index 1438ddcc293..9a36458bc9b 100644 --- a/src/common/expression/MatchPathPatternExpression.h +++ b/src/common/expression/MatchPathPatternExpression.h @@ -53,6 +53,10 @@ class MatchPathPatternExpression final : public Expression { return *matchPath_; } + MatchPath* matchPathPtr() const { + return matchPath_.get(); + } + private: friend ObjectPool; explicit MatchPathPatternExpression(ObjectPool* pool, std::unique_ptr&& matchPath) diff --git a/src/graph/context/ast/CypherAstContext.h b/src/graph/context/ast/CypherAstContext.h index e0fb879b2c9..dfba403e101 100644 --- a/src/graph/context/ast/CypherAstContext.h +++ b/src/graph/context/ast/CypherAstContext.h @@ -46,7 +46,7 @@ struct NodeInfo { struct EdgeInfo { bool anonymous{false}; - MatchStepRange* range{nullptr}; + std::unique_ptr range{nullptr}; std::vector edgeTypes; MatchEdge::Direction direction{MatchEdge::Direction::OUT_EDGE}; std::vector types; diff --git a/src/graph/executor/algo/ShortestPathBase.h b/src/graph/executor/algo/ShortestPathBase.h index 2a049246a74..d107f7fbbcd 100644 --- a/src/graph/executor/algo/ShortestPathBase.h +++ b/src/graph/executor/algo/ShortestPathBase.h @@ -23,7 +23,7 @@ class ShortestPathBase { std::unordered_map* stats) : pathNode_(node), qctx_(qctx), stats_(stats) { singleShortest_ = node->singleShortest(); - maxStep_ = node->stepRange()->max(); + maxStep_ = node->stepRange().max(); } virtual ~ShortestPathBase() {} diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp index f4b85fd6a8b..850820bf1fa 100644 --- a/src/graph/executor/query/TraverseExecutor.cpp +++ b/src/graph/executor/query/TraverseExecutor.cpp @@ -112,7 +112,7 @@ folly::Future TraverseExecutor::getNeighbors() { Expression* TraverseExecutor::selectFilter() { Expression* filter = nullptr; - if (!(currentStep_ == 1 && traverse_->zeroStep())) { + if (!(currentStep_ == 1 && range_.min() == 0)) { filter = const_cast(traverse_->filter()); } if (currentStep_ == 1) { @@ -166,18 +166,14 @@ folly::Future TraverseExecutor::handleResponse(RpcResponse&& resps) { initVertices_.emplace_back(vertex); } } - if (range_ && range_->min() == 0) { + if (range_.min() == 0) { result_.rows = buildZeroStepPath(); } } expand(iter.get()); if (!isFinalStep()) { if (vids_.empty()) { - if (range_ != nullptr) { - return buildResult(); - } else { - return folly::makeFuture(Status::OK()); - } + return buildResult(); } else { return getNeighbors(); } @@ -267,12 +263,8 @@ std::vector TraverseExecutor::buildZeroStepPath() { } folly::Future TraverseExecutor::buildResult() { - size_t minStep = 1; - size_t maxStep = 1; - if (range_ != nullptr) { - minStep = range_->min(); - maxStep = range_->max(); - } + size_t minStep = range_.min(); + size_t maxStep = range_.max(); result_.colNames = traverse_->colNames(); if (maxStep == 0) { diff --git a/src/graph/executor/query/TraverseExecutor.h b/src/graph/executor/query/TraverseExecutor.h index ed4ea9bf800..ecaa7e67fa2 100644 --- a/src/graph/executor/query/TraverseExecutor.h +++ b/src/graph/executor/query/TraverseExecutor.h @@ -67,8 +67,7 @@ class TraverseExecutor final : public StorageAccessExecutor { folly::Future buildPathMultiJobs(size_t minStep, size_t maxStep); bool isFinalStep() const { - return (range_ == nullptr && currentStep_ == 1) || - (range_ != nullptr && (currentStep_ == range_->max() || range_->max() == 0)); + return currentStep_ == range_.max() || range_.max() == 0; } bool filterSameEdge(const Row& lhs, @@ -133,7 +132,7 @@ class TraverseExecutor final : public StorageAccessExecutor { std::unordered_map, VertexHash, VertexEqual> adjList_; std::unordered_map, VertexHash, VertexEqual> dst2PathsMap_; const Traverse* traverse_{nullptr}; - MatchStepRange* range_{nullptr}; + MatchStepRange range_; size_t currentStep_{0}; }; diff --git a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp index cb7db5d8175..ae2421c811e 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp +++ b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp @@ -58,7 +58,8 @@ bool GetEdgesTransformAppendVerticesLimitRule::match(OptContext *ctx, if (colNames[colSize - 2][0] != '_') { // src return false; } - if (traverse->stepRange() != nullptr) { + const auto &stepRange = traverse->stepRange(); + if (stepRange.min() != 1 || stepRange.max() != 1) { return false; } // Can't apply vertex filter in GetEdges diff --git a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp index 17764389c2b..695ab2c4dfe 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp +++ b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp @@ -54,7 +54,8 @@ bool GetEdgesTransformRule::match(OptContext *ctx, const MatchedResult &matched) if (colNames[colSize - 2][0] != '_') { // src return false; } - if (traverse->stepRange() != nullptr) { + const auto &stepRange = traverse->stepRange(); + if (stepRange.min() != 1 || stepRange.max() != 1) { return false; } // Can't apply vertex filter in GetEdges diff --git a/src/graph/planner/match/MatchPathPlanner.cpp b/src/graph/planner/match/MatchPathPlanner.cpp index 129334967d9..741c18f785b 100644 --- a/src/graph/planner/match/MatchPathPlanner.cpp +++ b/src/graph/planner/match/MatchPathPlanner.cpp @@ -202,6 +202,10 @@ Status MatchPathPlanner::leftExpandFromNode(size_t startIndex, SubPlan& subplan) addNodeAlias(node); bool expandInto = isExpandInto(dst.alias); auto& edge = edgeInfos[i - 1]; + MatchStepRange stepRange(1, 1); + if (edge.range != nullptr) { + stepRange = *edge.range; + } auto traverse = Traverse::make(qctx, subplan.root, spaceId); traverse->setSrc(nextTraverseStart); auto vertexProps = SchemaUtil::getAllVertexProp(qctx, spaceId, true); @@ -212,7 +216,7 @@ Status MatchPathPlanner::leftExpandFromNode(size_t startIndex, SubPlan& subplan) traverse->setTagFilter(genVertexFilter(node)); traverse->setEdgeFilter(genEdgeFilter(edge)); traverse->setEdgeDirection(edge.direction); - traverse->setStepRange(edge.range); + traverse->setStepRange(stepRange); traverse->setDedup(); // If start from end of the path pattern, the first traverse would not // track the previous path, otherwise, it should. @@ -269,6 +273,10 @@ Status MatchPathPlanner::rightExpandFromNode(size_t startIndex, SubPlan& subplan bool expandInto = isExpandInto(dst.alias); auto& edge = edgeInfos[i]; + MatchStepRange stepRange(1, 1); + if (edge.range != nullptr) { + stepRange = *edge.range; + } auto traverse = Traverse::make(qctx, subplan.root, spaceId); traverse->setSrc(nextTraverseStart); auto vertexProps = SchemaUtil::getAllVertexProp(qctx, spaceId, true); @@ -279,7 +287,7 @@ Status MatchPathPlanner::rightExpandFromNode(size_t startIndex, SubPlan& subplan traverse->setTagFilter(genVertexFilter(node)); traverse->setEdgeFilter(genEdgeFilter(edge)); traverse->setEdgeDirection(edge.direction); - traverse->setStepRange(edge.range); + traverse->setStepRange(stepRange); traverse->setDedup(); traverse->setTrackPrevPath(i != startIndex); traverse->setColNames( diff --git a/src/graph/planner/match/ShortestPathPlanner.cpp b/src/graph/planner/match/ShortestPathPlanner.cpp index 41bfe4a9db5..0250e79c281 100644 --- a/src/graph/planner/match/ShortestPathPlanner.cpp +++ b/src/graph/planner/match/ShortestPathPlanner.cpp @@ -97,6 +97,11 @@ StatusOr ShortestPathPlanner::transform(WhereClauseContext* bindWhereCl auto cp = CrossJoin::make(qctx, leftPlan.root, rightPlan.root); + MatchStepRange stepRange(1, 1); + if (edge.range != nullptr) { + stepRange = *edge.range; + } + auto shortestPath = ShortestPath::make(qctx, cp, spaceId, singleShortest); auto vertexProp = SchemaUtil::getAllVertexProp(qctx, spaceId, true); NG_RETURN_IF_ERROR(vertexProp); @@ -104,7 +109,7 @@ StatusOr ShortestPathPlanner::transform(WhereClauseContext* bindWhereCl shortestPath->setEdgeProps(SchemaUtil::getEdgeProps(edge, false, qctx, spaceId)); shortestPath->setReverseEdgeProps(SchemaUtil::getEdgeProps(edge, true, qctx, spaceId)); shortestPath->setEdgeDirection(edge.direction); - shortestPath->setStepRange(edge.range); + shortestPath->setStepRange(stepRange); shortestPath->setColNames(std::move(colNames)); subplan.root = shortestPath; diff --git a/src/graph/planner/plan/Algo.cpp b/src/graph/planner/plan/Algo.cpp index 8416bddd65d..f2ef34a7180 100644 --- a/src/graph/planner/plan/Algo.cpp +++ b/src/graph/planner/plan/Algo.cpp @@ -39,7 +39,7 @@ std::unique_ptr ProduceAllPaths::explain() const { std::unique_ptr ShortestPath::explain() const { auto desc = SingleInputNode::explain(); addDescription("singleShortest", folly::toJson(util::toJson(singleShortest_)), desc.get()); - addDescription("steps", range_ != nullptr ? range_->toString() : "", desc.get()); + addDescription("steps", range_.toString(), desc.get()); addDescription("edgeDirection", apache::thrift::util::enumNameSafe(edgeDirection_), desc.get()); addDescription( "vertexProps", vertexProps_ ? folly::toJson(util::toJson(*vertexProps_)) : "", desc.get()); diff --git a/src/graph/planner/plan/Algo.h b/src/graph/planner/plan/Algo.h index ff50714817e..a58ae6f555a 100644 --- a/src/graph/planner/plan/Algo.h +++ b/src/graph/planner/plan/Algo.h @@ -174,7 +174,7 @@ class ShortestPath final : public SingleInputNode { std::unique_ptr explain() const override; - MatchStepRange* stepRange() const { + MatchStepRange stepRange() const { return range_; } @@ -202,7 +202,7 @@ class ShortestPath final : public SingleInputNode { return singleShortest_; } - void setStepRange(MatchStepRange* range) { + void setStepRange(const MatchStepRange& range) { range_ = range; } @@ -234,7 +234,7 @@ class ShortestPath final : public SingleInputNode { private: GraphSpaceID space_; bool singleShortest_{false}; - MatchStepRange* range_{nullptr}; + MatchStepRange range_; std::unique_ptr> edgeProps_; std::unique_ptr> reverseEdgeProps_; std::unique_ptr> vertexProps_; diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp index ad855bb112d..98383f09b37 100644 --- a/src/graph/planner/plan/Query.cpp +++ b/src/graph/planner/plan/Query.cpp @@ -790,7 +790,7 @@ void Traverse::cloneMembers(const Traverse& g) { std::unique_ptr Traverse::explain() const { auto desc = GetNeighbors::explain(); - addDescription("steps", range_ != nullptr ? range_->toString() : "", desc.get()); + addDescription("steps", range_.toString(), desc.get()); addDescription("vertex filter", vFilter_ != nullptr ? vFilter_->toString() : "", desc.get()); addDescription("edge filter", eFilter_ != nullptr ? eFilter_->toString() : "", desc.get()); addDescription("if_track_previous_path", folly::toJson(util::toJson(trackPrevPath_)), desc.get()); diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h index ded9e30e2c4..f485795f0c5 100644 --- a/src/graph/planner/plan/Query.h +++ b/src/graph/planner/plan/Query.h @@ -1581,17 +1581,17 @@ class Traverse final : public GetNeighbors { Traverse* clone() const override; - MatchStepRange* stepRange() const { + MatchStepRange stepRange() const { return range_; } bool isOneStep() const { - return !range_; + return range_.min() == 1 && range_.max() == 1; } // Contains zero step bool zeroStep() const { - return range_ != nullptr && range_->min() == 0; + return range_.min() == 0; } Expression* vFilter() const { @@ -1617,7 +1617,7 @@ class Traverse final : public GetNeighbors { return this->colNames().back(); } - void setStepRange(MatchStepRange* range) { + void setStepRange(const MatchStepRange& range) { range_ = range; } @@ -1659,7 +1659,7 @@ class Traverse final : public GetNeighbors { private: void cloneMembers(const Traverse& g); - MatchStepRange* range_{nullptr}; + MatchStepRange range_; Expression* vFilter_{nullptr}; Expression* eFilter_{nullptr}; bool trackPrevPath_{true}; diff --git a/src/graph/validator/MatchValidator.cpp b/src/graph/validator/MatchValidator.cpp index e47de251cf7..8c3e0ed7dd5 100644 --- a/src/graph/validator/MatchValidator.cpp +++ b/src/graph/validator/MatchValidator.cpp @@ -296,14 +296,14 @@ Status MatchValidator::buildEdgeInfo(const MatchPath *path, } } AliasType aliasType = AliasType::kEdge; - auto *stepRange = edge->range(); + auto stepRange = const_cast(edge)->range(); if (stepRange != nullptr) { NG_RETURN_IF_ERROR(validateStepRange(stepRange)); - edgeInfos[i].range = stepRange; // Type of [e*1..2], [e*2] should be inference to EdgeList if (stepRange->max() > stepRange->min() || stepRange->min() > 1) { aliasType = AliasType::kEdgeList; } + edgeInfos[i].range.reset(new MatchStepRange(*stepRange)); } if (alias.empty()) { anonymous = true; @@ -1100,36 +1100,36 @@ Status MatchValidator::validateMatchPathExpr( auto *matchPathExprImpl = const_cast( static_cast(matchPathExpr)); // Check variables - NG_RETURN_IF_ERROR(checkMatchPathExpr(matchPathExprImpl->matchPath(), availableAliases)); + NG_RETURN_IF_ERROR(checkMatchPathExpr(matchPathExprImpl->matchPathPtr(), availableAliases)); // Build path alias - auto &matchPath = matchPathExprImpl->matchPath(); - auto pathAlias = matchPath.toString(); - matchPath.setAlias(new std::string(pathAlias)); + auto matchPathPtr = matchPathExprImpl->matchPathPtr(); + auto pathAlias = matchPathPtr->toString(); + matchPathPtr->setAlias(new std::string(pathAlias)); if (matchPathExprImpl->genList() == nullptr) { // Don't done in expression visitor Expression *genList = InputPropertyExpression::make(pool, pathAlias); matchPathExprImpl->setGenList(genList); } paths.emplace_back(); - NG_RETURN_IF_ERROR(validatePath(&matchPath, paths.back())); - NG_RETURN_IF_ERROR(buildRollUpPathInfo(&matchPath, paths.back())); + NG_RETURN_IF_ERROR(validatePath(matchPathPtr, paths.back())); + NG_RETURN_IF_ERROR(buildRollUpPathInfo(matchPathPtr, paths.back())); } return Status::OK(); } -bool extractSinglePathPredicate(Expression *expr, std::vector &pathPreds) { +bool extractSinglePathPredicate(Expression *expr, std::vector &pathPreds) { if (expr->kind() == Expression::Kind::kMatchPathPattern) { - auto pred = static_cast(expr)->matchPath().clone(); - pred.setPredicate(); - pathPreds.emplace_back(std::move(pred)); + auto pred = static_cast(expr)->matchPathPtr(); + pred->setPredicate(); + pathPreds.emplace_back(pred); // Absorb expression into path predicate return true; } else if (expr->kind() == Expression::Kind::kUnaryNot) { auto *operand = static_cast(expr)->operand(); if (operand->kind() == Expression::Kind::kMatchPathPattern) { - auto pred = static_cast(operand)->matchPath().clone(); - pred.setAntiPredicate(); - pathPreds.emplace_back(std::move(pred)); + auto pred = static_cast(operand)->matchPathPtr(); + pred->setAntiPredicate(); + pathPreds.emplace_back(pred); // Absorb expression into path predicate return true; } else if (operand->kind() == Expression::Kind::kFunctionCall) { @@ -1138,9 +1138,9 @@ bool extractSinglePathPredicate(Expression *expr, std::vector &pathPr auto args = funcExpr->args()->args(); DCHECK_EQ(args.size(), 1); if (args[0]->kind() == Expression::Kind::kMatchPathPattern) { - auto pred = static_cast(args[0])->matchPath().clone(); - pred.setAntiPredicate(); - pathPreds.emplace_back(std::move(pred)); + auto pred = static_cast(args[0])->matchPathPtr(); + pred->setAntiPredicate(); + pathPreds.emplace_back(pred); // Absorb expression into path predicate return true; } @@ -1151,7 +1151,7 @@ bool extractSinglePathPredicate(Expression *expr, std::vector &pathPr return false; } -bool extractMultiPathPredicate(Expression *expr, std::vector &pathPreds) { +bool extractMultiPathPredicate(Expression *expr, std::vector &pathPreds) { if (expr->kind() == Expression::Kind::kLogicalAnd) { auto &operands = static_cast(expr)->operands(); for (auto iter = operands.begin(); iter != operands.end();) { @@ -1177,7 +1177,7 @@ Status MatchValidator::validatePathInWhere( auto *pool = qctx_->objPool(); ValidatePatternExpressionVisitor visitor(pool, vctx_); expr->accept(&visitor); - std::vector pathPreds; + std::vector pathPreds; // FIXME(czp): Delete this function and add new expression visitor to cover all general cases if (extractMultiPathPredicate(expr, pathPreds)) { wctx.filter = nullptr; @@ -1190,11 +1190,11 @@ Status MatchValidator::validatePathInWhere( for (auto &pred : pathPreds) { NG_RETURN_IF_ERROR(checkMatchPathExpr(pred, availableAliases)); // Build path alias - auto pathAlias = pred.toString(); - pred.setAlias(new std::string(pathAlias)); + auto pathAlias = pred->toString(); + pred->setAlias(new std::string(pathAlias)); paths.emplace_back(); - NG_RETURN_IF_ERROR(validatePath(&pred, paths.back())); - NG_RETURN_IF_ERROR(buildRollUpPathInfo(&pred, paths.back())); + NG_RETURN_IF_ERROR(validatePath(pred, paths.back())); + NG_RETURN_IF_ERROR(buildRollUpPathInfo(pred, paths.back())); } // All inside pattern expressions are path predicate @@ -1211,7 +1211,7 @@ Status MatchValidator::validatePathInWhere( auto *matchPathExprImpl = const_cast( static_cast(matchPathExpr)); // Check variables - NG_RETURN_IF_ERROR(checkMatchPathExpr(matchPathExprImpl->matchPath(), availableAliases)); + NG_RETURN_IF_ERROR(checkMatchPathExpr(matchPathExprImpl->matchPathPtr(), availableAliases)); // Build path alias auto &matchPath = matchPathExprImpl->matchPath(); auto pathAlias = matchPath.toString(); @@ -1230,22 +1230,21 @@ Status MatchValidator::validatePathInWhere( } /*static*/ Status MatchValidator::checkMatchPathExpr( - const MatchPath &matchPath, - const std::unordered_map &availableAliases) { - if (matchPath.alias() != nullptr) { - const auto find = availableAliases.find(*matchPath.alias()); + MatchPath *matchPath, const std::unordered_map &availableAliases) { + if (matchPath->alias() != nullptr) { + const auto find = availableAliases.find(*matchPath->alias()); if (find == availableAliases.end()) { return Status::SemanticError( "PatternExpression are not allowed to introduce new variables: `%s'.", - matchPath.alias()->c_str()); + matchPath->alias()->c_str()); } if (find->second != AliasType::kPath) { return Status::SemanticError("`%s' is defined with type %s, but referenced with type Path", - matchPath.alias()->c_str(), + matchPath->alias()->c_str(), AliasTypeName::get(find->second).c_str()); } } - for (const auto &node : matchPath.nodes()) { + for (const auto &node : matchPath->nodes()) { if (node->variableDefinedSource() == MatchNode::VariableDefinedSource::kExpression) { // Checked in visitor continue; @@ -1264,7 +1263,7 @@ Status MatchValidator::validatePathInWhere( } } } - for (const auto &edge : matchPath.edges()) { + for (const auto &edge : matchPath->edges()) { if (!edge->alias().empty()) { const auto find = availableAliases.find(edge->alias()); if (find == availableAliases.end()) { diff --git a/src/graph/validator/MatchValidator.h b/src/graph/validator/MatchValidator.h index f1e9f5166d5..c78de9d3e7a 100644 --- a/src/graph/validator/MatchValidator.h +++ b/src/graph/validator/MatchValidator.h @@ -104,8 +104,7 @@ class MatchValidator final : public Validator { std::vector &paths); static Status checkMatchPathExpr( - const MatchPath &matchPath, - const std::unordered_map &availableAliases); + MatchPath *matchPath, const std::unordered_map &availableAliases); static Status buildRollUpPathInfo(const MatchPath *path, Path &pathInfo); diff --git a/src/parser/MatchPath.h b/src/parser/MatchPath.h index a0154f6f022..487ad0dba8b 100644 --- a/src/parser/MatchPath.h +++ b/src/parser/MatchPath.h @@ -111,7 +111,7 @@ class MatchEdge final { return props_; } - auto* range() const { + MatchStepRange* range() const { return range_.get(); } diff --git a/tests/tck/features/match/PathExpr.feature b/tests/tck/features/match/PathExpr.feature index d115285b67d..369db9b86b4 100644 --- a/tests/tck/features/match/PathExpr.feature +++ b/tests/tck/features/match/PathExpr.feature @@ -362,3 +362,88 @@ Feature: Basic match | p | | <("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})> | | <("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})> | + + Scenario: pattern in where + When executing query: + """ + MATCH (v:player)-[e]->(b) + WHERE id(v) IN ['Tim Duncan', 'Tony Parker'] AND (b)-[*1..2]->(v) + RETURN b + """ + Then the result should be, in any order: + | b | + | ("Danny Green" :player{age: 31, name: "Danny Green"}) | + | ("LaMarcus Aldridge" :player{age: 33, name: "LaMarcus Aldridge"}) | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | + | ("LaMarcus Aldridge" :player{age: 33, name: "LaMarcus Aldridge"}) | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + | ("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"}) | + | ("LaMarcus Aldridge" :player{age: 33, name: "LaMarcus Aldridge"}) | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + | ("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"}) | + When executing query: + """ + MATCH (v:player)-[e:like]->(b) + WHERE (b)-[:teammate]->(v) + RETURN e + """ + Then the result should be, in any order: + | e | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | [:like "Danny Green"->"Tim Duncan" @0 {likeness: 70}] | + | [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}] | + | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | [:like "LaMarcus Aldridge"->"Tim Duncan" @0 {likeness: 75}] | + | [:like "LaMarcus Aldridge"->"Tony Parker" @0 {likeness: 75}] | + When executing query: + """ + MATCH (v:player)-[e:like*2]->(b) + WHERE id(v) == 'Tony Parker' AND (b)-[*1..2]->(v) + RETURN distinct e + """ + Then the result should be, in any order: + | e | + | [[:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}], [:like "LaMarcus Aldridge"->"Tim Duncan" @0 {likeness: 75}]] | + | [[:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}], [:like "LaMarcus Aldridge"->"Tony Parker" @0 {likeness: 75}]] | + | [[:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}], [:like "Manu Ginobili"->"Tim Duncan" @0 {likeness: 90}]] | + | [[:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}], [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}]] | + | [[:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}], [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}]] | + When executing query: + """ + MATCH (v:player{name: 'Tim Duncan'})-[e:like*3]->(n), (t:team {name: "Spurs"}) + WITH v, e, collect(distinct n) AS ns + UNWIND [n in ns | ()-[e*3]->(n:player)] AS p + RETURN p + """ + Then the result should be, in any order: + | p | + | [] | + | [] | + | [] | + | [] | + | [] | + When executing query: + """ + MATCH (v:player)-[e:like*3]->(n) + WHERE (n)-[e*3]->(:player) + RETURN v + """ + Then the result should be, in any order: + | v | + When executing query: + """ + MATCH (v:player)-[e:like*1..3]->(n) WHERE (n)-[e*1..4]->(:player) return v + """ + Then the result should be, in any order: + | v | + When executing query: + """ + MATCH (v:player)-[e:like*3]->(n) WHERE id(v)=="Tim Duncan" and (n)-[e*3]->(:player) return v + """ + Then the result should be, in any order: + | v | diff --git a/tests/tck/features/match/PathExprRefLocalVariable.feature b/tests/tck/features/match/PathExprRefLocalVariable.feature index 02355c3d134..5790242cdd7 100644 --- a/tests/tck/features/match/PathExprRefLocalVariable.feature +++ b/tests/tck/features/match/PathExprRefLocalVariable.feature @@ -123,21 +123,6 @@ Feature: Path expression reference local defined variables | "Rajon Rondo" | | "Ray Allen" | - @skip - Scenario: Fix after issue #2045 - When executing query: - """ - MATCH (v:player)-[e:like*1..3]->(n) WHERE (n)-[e*1..4]->(:player) return v - """ - Then the result should be, in any order: - | v | - When executing query: - """ - MATCH (v:player)-[e:like*3]->(n) WHERE id(v)=="Tim Duncan" and (n)-[e*3]->(:player) return v - """ - Then the result should be, in any order: - | v | - Scenario: In With When executing query: """ diff --git a/tests/tck/features/match/VariableLengthPattern.feature b/tests/tck/features/match/VariableLengthPattern.feature index a8e74e88ea5..5563b220f2c 100644 --- a/tests/tck/features/match/VariableLengthPattern.feature +++ b/tests/tck/features/match/VariableLengthPattern.feature @@ -478,6 +478,7 @@ Feature: Variable length Pattern match (m to n) Then the result should be, in any order: | cnt | + @skip Scenario: variable pattern in where clause When executing query: """ @@ -487,7 +488,7 @@ Feature: Variable length Pattern match (m to n) """ Then the result should be, in any order: | cnt | - | 182 | + | 76 | When executing query: """ MATCH (v:player{name: 'Tim Duncan'})-[e*0..2]-(v2) From b0b84df4a00f46777658b57dba950dfebf9c9a38 Mon Sep 17 00:00:00 2001 From: haowen <19355821+wenhaocs@users.noreply.github.com> Date: Tue, 10 Jan 2023 20:47:03 -0800 Subject: [PATCH 29/51] Update response message when adding schema historically existed (#5227) * update the error code and message for checking history schemas * update tck * update comment * change to log error * fix ddl tck * increase wait time in schema.feature Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/clients/meta/MetaClient.cpp | 2 ++ src/common/graph/Response.h | 1 + src/interface/common.thrift | 1 + src/meta/MetaServiceUtils.cpp | 13 +++++++++++-- tests/tck/features/ddl/Ddl.feature | 4 ++-- tests/tck/features/schema/Schema.feature | 6 +++--- 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index 7446ab95f12..e04cb91214e 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -828,6 +828,8 @@ Status MetaClient::handleResponse(const RESP& resp) { return Status::Error("No hosts!"); case nebula::cpp2::ErrorCode::E_EXISTED: return Status::Error("Existed!"); + case nebula::cpp2::ErrorCode::E_HISTORY_CONFLICT: + return Status::Error("Schema exisited before!"); case nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND: return Status::SpaceNotFound("Space not existed!"); case nebula::cpp2::ErrorCode::E_TAG_NOT_FOUND: diff --git a/src/common/graph/Response.h b/src/common/graph/Response.h index 7989548d565..8ce83a78bf6 100644 --- a/src/common/graph/Response.h +++ b/src/common/graph/Response.h @@ -77,6 +77,7 @@ X(E_WRONGCLUSTER, -2010) \ X(E_ZONE_NOT_ENOUGH, -2011) \ X(E_ZONE_IS_EMPTY, -2012) \ + X(E_HISTORY_CONFLICT, -2018) \ \ X(E_STORE_FAILURE, -2021) \ X(E_STORE_SEGMENT_ILLEGAL, -2022) \ diff --git a/src/interface/common.thrift b/src/interface/common.thrift index 0bdf5d44f94..7a87b1fb807 100644 --- a/src/interface/common.thrift +++ b/src/interface/common.thrift @@ -376,6 +376,7 @@ enum ErrorCode { E_RELATED_INDEX_EXISTS = -2015, // There are still indexes related to tag or edge, cannot drop it E_RELATED_SPACE_EXISTS = -2016, // There are still some space on the host, cannot drop it E_RELATED_FULLTEXT_INDEX_EXISTS = -2017, // There are still fulltext index on tag/edge + E_HISTORY_CONFLICT = -2018, // Existed before (e.g., schema) E_STORE_FAILURE = -2021, // Failed to store data E_STORE_SEGMENT_ILLEGAL = -2022, // Illegal storage segment diff --git a/src/meta/MetaServiceUtils.cpp b/src/meta/MetaServiceUtils.cpp index 4a23439dc1e..6e864731a53 100644 --- a/src/meta/MetaServiceUtils.cpp +++ b/src/meta/MetaServiceUtils.cpp @@ -87,11 +87,20 @@ nebula::cpp2::ErrorCode MetaServiceUtils::alterColumnDefs( bool isEdge) { switch (op) { case cpp2::AlterSchemaOp::ADD: + // Check the current schema first. Then check all schemas. + for (auto it = cols.begin(); it != cols.end(); ++it) { + if (it->get_name() == col.get_name()) { + LOG(ERROR) << "Column existing: " << col.get_name(); + return nebula::cpp2::ErrorCode::E_EXISTED; + } + } + // There won't any two columns having the same name across all schemas. If there is a column + // having the same name with the intended change, it must be from history schemas. for (auto& versionedCols : allVersionedCols) { for (auto it = versionedCols.begin(); it != versionedCols.end(); ++it) { if (it->get_name() == col.get_name()) { - LOG(ERROR) << "Column currently or previously existing: " << col.get_name(); - return nebula::cpp2::ErrorCode::E_EXISTED; + LOG(ERROR) << "Column previously existing: " << col.get_name(); + return nebula::cpp2::ErrorCode::E_HISTORY_CONFLICT; } } } diff --git a/tests/tck/features/ddl/Ddl.feature b/tests/tck/features/ddl/Ddl.feature index bb9eb495fa8..70083f4c864 100644 --- a/tests/tck/features/ddl/Ddl.feature +++ b/tests/tck/features/ddl/Ddl.feature @@ -83,7 +83,7 @@ Feature: DDL test ALTER TAG B ADD (name string) """ # IMHO, this is really confusing. https://github.com/vesoft-inc/nebula/issues/2671 - Then a ExecutionError should be raised at runtime: Existed! + Then a ExecutionError should be raised at runtime: Schema exisited before! When executing query: """ ALTER TAG B ADD (namex string) @@ -225,7 +225,7 @@ Feature: DDL test """ ALTER EDGE E2 ADD (name string) """ - Then a ExecutionError should be raised at runtime: Existed! + Then a ExecutionError should be raised at runtime: Schema exisited before! When executing query: """ ALTER EDGE E2 ADD (namex string) diff --git a/tests/tck/features/schema/Schema.feature b/tests/tck/features/schema/Schema.feature index 6ac6c6f35ac..fc386184eaf 100644 --- a/tests/tck/features/schema/Schema.feature +++ b/tests/tck/features/schema/Schema.feature @@ -956,7 +956,7 @@ Feature: Insert string vid of vertex and edge CREATE TAG person(name string, age int); """ Then the execution should be successful - And wait 3 seconds + And wait 10 seconds When executing query: """ INSERT VERTEX person values "1":("Tom", 23); @@ -974,7 +974,7 @@ Feature: Insert string vid of vertex and edge ALTER TAG person DROP (age); """ Then the execution should be successful - And wait 3 seconds + And wait 10 seconds When executing query: """ FETCH PROP ON person "1" yield properties(vertex) AS props; @@ -986,4 +986,4 @@ Feature: Insert string vid of vertex and edge """ ALTER TAG person ADD (age int); """ - Then a ExecutionError should be raised at runtime: Existed + Then a ExecutionError should be raised at runtime: Schema exisited before! From 992af5de7692554ad418c97d56ab74b95c01bc2e Mon Sep 17 00:00:00 2001 From: Alex Xing <90179377+SuperYoko@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:41:37 +0800 Subject: [PATCH 30/51] fix error code (#5233) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/meta/processors/job/RebuildJobExecutor.cpp | 4 ++-- src/meta/processors/job/StatsJobExecutor.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/meta/processors/job/RebuildJobExecutor.cpp b/src/meta/processors/job/RebuildJobExecutor.cpp index 0627416b820..a485144151c 100644 --- a/src/meta/processors/job/RebuildJobExecutor.cpp +++ b/src/meta/processors/job/RebuildJobExecutor.cpp @@ -64,12 +64,12 @@ nebula::cpp2::ErrorCode RebuildJobExecutor::stop() { auto tries = folly::collectAll(std::move(futures)).get(); if (std::any_of(tries.begin(), tries.end(), [](auto& t) { return t.hasException(); })) { LOG(INFO) << "RebuildJobExecutor::stop() RPC failure."; - return nebula::cpp2::ErrorCode::E_BALANCER_FAILURE; + return nebula::cpp2::ErrorCode::E_RPC_FAILURE; } for (const auto& t : tries) { if (!t.value().ok()) { LOG(INFO) << "Stop Build Index Failed"; - return nebula::cpp2::ErrorCode::E_BALANCER_FAILURE; + return nebula::cpp2::ErrorCode::E_RPC_FAILURE; } } return nebula::cpp2::ErrorCode::SUCCEEDED; diff --git a/src/meta/processors/job/StatsJobExecutor.cpp b/src/meta/processors/job/StatsJobExecutor.cpp index b381172d5b1..4d6aa26c413 100644 --- a/src/meta/processors/job/StatsJobExecutor.cpp +++ b/src/meta/processors/job/StatsJobExecutor.cpp @@ -198,13 +198,13 @@ nebula::cpp2::ErrorCode StatsJobExecutor::stop() { auto tries = folly::collectAll(std::move(futures)).get(); if (std::any_of(tries.begin(), tries.end(), [](auto& t) { return t.hasException(); })) { LOG(INFO) << "stats job stop() RPC failure."; - return nebula::cpp2::ErrorCode::E_BALANCER_FAILURE; + return nebula::cpp2::ErrorCode::E_RPC_FAILURE; } for (const auto& t : tries) { if (!t.value().ok()) { LOG(INFO) << "Stop stats job Failed"; - return nebula::cpp2::ErrorCode::E_BALANCER_FAILURE; + return nebula::cpp2::ErrorCode::E_RPC_FAILURE; } } return nebula::cpp2::ErrorCode::SUCCEEDED; From 9c14be2efc655c803b3577c5cadf1cc50e245cf9 Mon Sep 17 00:00:00 2001 From: codesigner Date: Wed, 11 Jan 2023 16:55:16 +0800 Subject: [PATCH 31/51] print memory stats default to false (#5234) * print memory stats default to false * update conf Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- conf/nebula-graphd.conf.default | 2 +- conf/nebula-graphd.conf.production | 2 +- conf/nebula-storaged.conf.default | 2 +- conf/nebula-storaged.conf.production | 2 +- src/common/memory/MemoryUtils.cpp | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default index 871ddd667c3..c1be4f72580 100644 --- a/conf/nebula-graphd.conf.default +++ b/conf/nebula-graphd.conf.default @@ -105,7 +105,7 @@ --memory_tracker_untracked_reserved_memory_mb=50 # enable log memory tracker stats periodically ---memory_tracker_detail_log=true +--memory_tracker_detail_log=false # log memory tacker stats interval in milliseconds --memory_tracker_detail_log_interval_ms=60000 diff --git a/conf/nebula-graphd.conf.production b/conf/nebula-graphd.conf.production index ad28a43cf39..3f1d828d533 100644 --- a/conf/nebula-graphd.conf.production +++ b/conf/nebula-graphd.conf.production @@ -104,7 +104,7 @@ --memory_tracker_untracked_reserved_memory_mb=50 # enable log memory tracker stats periodically ---memory_tracker_detail_log=true +--memory_tracker_detail_log=false # log memory tacker stats interval in milliseconds --memory_tracker_detail_log_interval_ms=60000 diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default index 4d102e616a8..7746aef5057 100644 --- a/conf/nebula-storaged.conf.default +++ b/conf/nebula-storaged.conf.default @@ -130,7 +130,7 @@ --memory_tracker_untracked_reserved_memory_mb=50 # enable log memory tracker stats periodically ---memory_tracker_detail_log=true +--memory_tracker_detail_log=false # log memory tacker stats interval in milliseconds --memory_tracker_detail_log_interval_ms=60000 diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production index db71776e61f..a5f3f64b92a 100644 --- a/conf/nebula-storaged.conf.production +++ b/conf/nebula-storaged.conf.production @@ -131,7 +131,7 @@ --memory_tracker_untracked_reserved_memory_mb=50 # enable log memory tracker stats periodically ---memory_tracker_detail_log=true +--memory_tracker_detail_log=false # log memory tacker stats interval in milliseconds --memory_tracker_detail_log_interval_ms=60000 diff --git a/src/common/memory/MemoryUtils.cpp b/src/common/memory/MemoryUtils.cpp index 1d7170baffb..f8950fd3d21 100644 --- a/src/common/memory/MemoryUtils.cpp +++ b/src/common/memory/MemoryUtils.cpp @@ -44,7 +44,7 @@ DEFINE_string(cgroup_v2_memory_current_path, DEFINE_bool(memory_purge_enabled, true, "memory purge enabled, default true"); DEFINE_int32(memory_purge_interval_seconds, 10, "memory purge interval in seconds, default 10"); -DEFINE_bool(memory_tracker_detail_log, true, "print memory stats detail log"); +DEFINE_bool(memory_tracker_detail_log, false, "print memory stats detail log"); DEFINE_int32(memory_tracker_detail_log_interval_ms, 60000, "print memory stats detail log interval in ms"); From 306f5da27b4cdc453223d1161a72c648a8edbcdc Mon Sep 17 00:00:00 2001 From: "jie.wang" <38901892+jievince@users.noreply.github.com> Date: Wed, 11 Jan 2023 23:43:46 +0800 Subject: [PATCH 32/51] fix bug of extract prop expr visitor (#5238) --- src/graph/validator/GoValidator.cpp | 9 +++--- src/graph/validator/GoValidator.h | 2 +- src/graph/visitor/ExtractPropExprVisitor.cpp | 4 --- src/graph/visitor/ExtractPropExprVisitor.h | 2 -- tests/tck/features/go/GO.feature | 31 ++++++++++++++++++++ 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/graph/validator/GoValidator.cpp b/src/graph/validator/GoValidator.cpp index cb2ebf62cd1..065db1ab2f8 100644 --- a/src/graph/validator/GoValidator.cpp +++ b/src/graph/validator/GoValidator.cpp @@ -188,8 +188,8 @@ Status GoValidator::extractTagIds() { return Status::OK(); } -void GoValidator::extractPropExprs(const Expression* expr, - std::unordered_set& uniqueExpr) { +Status GoValidator::extractPropExprs(const Expression* expr, + std::unordered_set& uniqueExpr) { ExtractPropExprVisitor visitor(vctx_, goCtx_->srcEdgePropsExpr, goCtx_->dstPropsExpr, @@ -197,6 +197,7 @@ void GoValidator::extractPropExprs(const Expression* expr, propExprColMap_, uniqueExpr); const_cast(expr)->accept(&visitor); + return std::move(visitor).status(); } Expression* GoValidator::rewriteVertexEdge2EdgeProp(const Expression* expr) { @@ -277,13 +278,13 @@ Status GoValidator::buildColumns() { std::unordered_set uniqueEdgeVertexExpr; auto filter = goCtx_->filter; if (filter != nullptr) { - extractPropExprs(filter, uniqueEdgeVertexExpr); + NG_RETURN_IF_ERROR(extractPropExprs(filter, uniqueEdgeVertexExpr)); goCtx_->filter = rewrite2VarProp(filter); } auto* newYieldExpr = pool->makeAndAdd(); for (auto* col : goCtx_->yieldExpr->columns()) { - extractPropExprs(col->expr(), uniqueEdgeVertexExpr); + NG_RETURN_IF_ERROR(extractPropExprs(col->expr(), uniqueEdgeVertexExpr)); newYieldExpr->addColumn(new YieldColumn(rewrite2VarProp(col->expr()), col->alias())); } diff --git a/src/graph/validator/GoValidator.h b/src/graph/validator/GoValidator.h index 8299c74b569..0f5293f7021 100644 --- a/src/graph/validator/GoValidator.h +++ b/src/graph/validator/GoValidator.h @@ -36,7 +36,7 @@ class GoValidator final : public Validator { Status extractTagIds(); - void extractPropExprs(const Expression* expr, std::unordered_set& uniqueCols); + Status extractPropExprs(const Expression* expr, std::unordered_set& uniqueCols); Expression* rewrite2VarProp(const Expression* expr); diff --git a/src/graph/visitor/ExtractPropExprVisitor.cpp b/src/graph/visitor/ExtractPropExprVisitor.cpp index 61e79c11c61..fa2dd1db8b8 100644 --- a/src/graph/visitor/ExtractPropExprVisitor.cpp +++ b/src/graph/visitor/ExtractPropExprVisitor.cpp @@ -47,10 +47,6 @@ void ExtractPropExprVisitor::visit(VariableExpression* expr) { UNUSED(expr); } -void ExtractPropExprVisitor::visit(SubscriptExpression* expr) { - reportError(expr); -} - void ExtractPropExprVisitor::visit(LabelExpression* expr) { reportError(expr); } diff --git a/src/graph/visitor/ExtractPropExprVisitor.h b/src/graph/visitor/ExtractPropExprVisitor.h index 87095bd8db6..470d403d4d0 100644 --- a/src/graph/visitor/ExtractPropExprVisitor.h +++ b/src/graph/visitor/ExtractPropExprVisitor.h @@ -59,8 +59,6 @@ class ExtractPropExprVisitor final : public ExprVisitorImpl { // vertex/edge expression void visit(VertexExpression *) override; void visit(EdgeExpression *) override; - // binary expression - void visit(SubscriptExpression *) override; // column expression void visit(ColumnExpression *) override; diff --git a/tests/tck/features/go/GO.feature b/tests/tck/features/go/GO.feature index 911d9f4b4cc..9948f3eb869 100644 --- a/tests/tck/features/go/GO.feature +++ b/tests/tck/features/go/GO.feature @@ -576,6 +576,37 @@ Feature: Go Sentence | "Chris Paul" | "Carmelo Anthony" | "Dwyane Wade" | | "Chris Paul" | "Dwyane Wade" | "LeBron James" | | "Chris Paul" | "Dwyane Wade" | "Carmelo Anthony" | + When executing query: + """ + GO 3 STEPS FROM "Tim Duncan" OVER like,serve + WHERE (serve.start_year>2000 OR like.likeness>90) AND size(labels($$)[0]) > 0 AND $$.player.age>40 + yield $$ as v + """ + Then the result should be, in any order, with relax comparison: + | v | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + When executing query: + """ + GO 3 STEPS FROM "Tim Duncan" OVER like,serve + WHERE size(labels($$))>0 AND $$.player.age>40 + yield $$ as v + """ + Then the result should be, in any order, with relax comparison: + | v | + | ("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"}) | + | ("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"}) | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + When executing query: + """ + GO 3 STEPS FROM "Tim Duncan" OVER like,serve + WHERE $$.player.age>40 AND size(labels($$)[0]) > 0 + yield $$ as v + """ + Then the result should be, in any order, with relax comparison: + | v | + | ("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"}) | + | ("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"}) | + | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | @skip Scenario: all prop(reason = $-.* over * $var.* not implement) From c953cf9993bed7f8bf7c7047f154d403bc0a6d65 Mon Sep 17 00:00:00 2001 From: "jie.wang" <38901892+jievince@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:59:56 +0800 Subject: [PATCH 33/51] forbid invalid prop expr used in cypher (#5242) --- .../executor/algo/BFSShortestPathExecutor.cpp | 16 +++--- .../executor/algo/ProduceAllPathsExecutor.cpp | 24 ++++---- src/graph/executor/maintain/EdgeExecutor.cpp | 7 +-- .../executor/maintain/EdgeIndexExecutor.cpp | 14 ++--- src/graph/validator/MatchValidator.cpp | 26 ++++++--- tests/tck/features/match/Base.feature | 55 +++++++++++++++++++ 6 files changed, 97 insertions(+), 45 deletions(-) diff --git a/src/graph/executor/algo/BFSShortestPathExecutor.cpp b/src/graph/executor/algo/BFSShortestPathExecutor.cpp index cf9faafa0d9..098ad840df5 100644 --- a/src/graph/executor/algo/BFSShortestPathExecutor.cpp +++ b/src/graph/executor/algo/BFSShortestPathExecutor.cpp @@ -167,15 +167,13 @@ folly::Future BFSShortestPathExecutor::conjunctPath() { } } - return folly::collect(futures) - .via(runner()) - .thenValue([this](auto&& resps) { - memory::MemoryCheckGuard guard; - for (auto& resp : resps) { - currentDs_.append(std::move(resp)); - } - return Status::OK(); - }); + return folly::collect(futures).via(runner()).thenValue([this](auto&& resps) { + memory::MemoryCheckGuard guard; + for (auto& resp : resps) { + currentDs_.append(std::move(resp)); + } + return Status::OK(); + }); } DataSet BFSShortestPathExecutor::doConjunct(const std::vector& meetVids, diff --git a/src/graph/executor/algo/ProduceAllPathsExecutor.cpp b/src/graph/executor/algo/ProduceAllPathsExecutor.cpp index f2b709a000f..0011caa5638 100644 --- a/src/graph/executor/algo/ProduceAllPathsExecutor.cpp +++ b/src/graph/executor/algo/ProduceAllPathsExecutor.cpp @@ -194,19 +194,17 @@ folly::Future ProduceAllPathsExecutor::conjunctPath() { } } - return folly::collect(futures) - .via(runner()) - .thenValue([this](auto&& resps) { - memory::MemoryCheckGuard guard; - for (auto& resp : resps) { - currentDs_.append(std::move(resp)); - } - preLeftPaths_.swap(leftPaths_); - preRightPaths_.swap(rightPaths_); - leftPaths_.clear(); - rightPaths_.clear(); - return Status::OK(); - }); + return folly::collect(futures).via(runner()).thenValue([this](auto&& resps) { + memory::MemoryCheckGuard guard; + for (auto& resp : resps) { + currentDs_.append(std::move(resp)); + } + preLeftPaths_.swap(leftPaths_); + preRightPaths_.swap(rightPaths_); + leftPaths_.clear(); + rightPaths_.clear(); + return Status::OK(); + }); } void ProduceAllPathsExecutor::setNextStepVid(Interims& paths, const string& var) { diff --git a/src/graph/executor/maintain/EdgeExecutor.cpp b/src/graph/executor/maintain/EdgeExecutor.cpp index 1c8a3151f44..d790e085f1d 100644 --- a/src/graph/executor/maintain/EdgeExecutor.cpp +++ b/src/graph/executor/maintain/EdgeExecutor.cpp @@ -86,11 +86,8 @@ folly::Future ShowEdgesExecutor::execute() { SCOPED_TIMER(&execTime_); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listEdgeSchemas(spaceId) - .via(runner()) - .thenValue([this, spaceId](StatusOr> resp) { + return qctx()->getMetaClient()->listEdgeSchemas(spaceId).via(runner()).thenValue( + [this, spaceId](StatusOr> resp) { memory::MemoryCheckGuard guard; if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Show edges failed: " << resp.status(); diff --git a/src/graph/executor/maintain/EdgeIndexExecutor.cpp b/src/graph/executor/maintain/EdgeIndexExecutor.cpp index 00f1f2e9d20..8fc1da1114a 100644 --- a/src/graph/executor/maintain/EdgeIndexExecutor.cpp +++ b/src/graph/executor/maintain/EdgeIndexExecutor.cpp @@ -118,11 +118,8 @@ folly::Future ShowEdgeIndexesExecutor::execute() { auto *iNode = asNode(node()); const auto &bySchema = iNode->name(); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listEdgeIndexes(spaceId) - .via(runner()) - .thenValue([this, spaceId, bySchema](StatusOr> resp) { + return qctx()->getMetaClient()->listEdgeIndexes(spaceId).via(runner()).thenValue( + [this, spaceId, bySchema](StatusOr> resp) { memory::MemoryCheckGuard guard; if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Show edge indexes failed" << resp.status(); @@ -174,11 +171,8 @@ folly::Future ShowEdgeIndexStatusExecutor::execute() { SCOPED_TIMER(&execTime_); auto spaceId = qctx()->rctx()->session()->space().id; - return qctx() - ->getMetaClient() - ->listEdgeIndexStatus(spaceId) - .via(runner()) - .thenValue([this, spaceId](StatusOr> resp) { + return qctx()->getMetaClient()->listEdgeIndexStatus(spaceId).via(runner()).thenValue( + [this, spaceId](StatusOr> resp) { memory::MemoryCheckGuard guard; if (!resp.ok()) { LOG(WARNING) << "SpaceId: " << spaceId << ", Show edge index status failed" diff --git a/src/graph/validator/MatchValidator.cpp b/src/graph/validator/MatchValidator.cpp index 8c3e0ed7dd5..18d6f09754f 100644 --- a/src/graph/validator/MatchValidator.cpp +++ b/src/graph/validator/MatchValidator.cpp @@ -509,14 +509,19 @@ Status MatchValidator::validateReturn(MatchReturn *ret, Status MatchValidator::validateAliases( const std::vector &exprs, const std::unordered_map &aliasesAvailable) const { - static const std::unordered_set kinds = {Expression::Kind::kLabel, - Expression::Kind::kLabelAttribute, - Expression::Kind::kLabelTagProperty, - // primitive props - Expression::Kind::kEdgeSrc, - Expression::Kind::kEdgeDst, - Expression::Kind::kEdgeRank, - Expression::Kind::kEdgeType}; + static const std::unordered_set kinds = { + Expression::Kind::kLabel, + Expression::Kind::kLabelAttribute, + Expression::Kind::kLabelTagProperty, + // primitive props + Expression::Kind::kEdgeSrc, + Expression::Kind::kEdgeDst, + Expression::Kind::kEdgeRank, + Expression::Kind::kEdgeType, + // invalid prop exprs + Expression::Kind::kSrcProperty, + Expression::Kind::kDstProperty, + }; for (auto *expr : exprs) { auto refExprs = ExpressionUtils::collectAll(expr, kinds); @@ -1058,6 +1063,11 @@ Status MatchValidator::checkAlias( name.c_str()); } } + case Expression::Kind::kSrcProperty: + case Expression::Kind::kDstProperty: { + return Status::SemanticError("Expression %s is not allowed to use in cypher", + refExpr->toString().c_str()); + } default: // refExpr must satisfy one of cases and should never hit this branch break; } diff --git a/tests/tck/features/match/Base.feature b/tests/tck/features/match/Base.feature index f8c0f8c07b1..9355b90eb8a 100644 --- a/tests/tck/features/match/Base.feature +++ b/tests/tck/features/match/Base.feature @@ -1072,6 +1072,61 @@ Feature: Basic match MATCH (v{name: "Tim Duncan"}) return v """ Then a SemanticError should be raised at runtime: `name:"Tim Duncan"': No tag found for property. + When executing query: + """ + MATCH (v:player) RETURN $$.player.age AS a + """ + Then a SemanticError should be raised at runtime: Expression $$.player.age is not allowed to use in cypher + When executing query: + """ + MATCH (v:player) RETURN $^.player.age AS a + """ + Then a SemanticError should be raised at runtime: Expression $^.player.age is not allowed to use in cypher + When executing query: + """ + MATCH (v:player) RETURN $-.player.age AS a + """ + Then a SemanticError should be raised at runtime: `$-.player', not exist prop `player' + When executing query: + """ + MATCH (v:player) WHERE $var.player > 0 RETURN v + """ + Then a SemanticError should be raised at runtime: `$var.player', not exist variable `var' + When executing query: + """ + MATCH (v:player) WHERE $$.player.age > 0 RETURN v + """ + Then a SemanticError should be raised at runtime: Expression $$.player.age is not allowed to use in cypher + When executing query: + """ + MATCH (v:player) WHERE $^.player.age > 0 RETURN v + """ + Then a SemanticError should be raised at runtime: Expression $^.player.age is not allowed to use in cypher + When executing query: + """ + MATCH (v:player) WHERE $-.player.age > 0 RETURN v + """ + Then a SemanticError should be raised at runtime: `$-.player', not exist prop `player' + When executing query: + """ + MATCH (v:player) WHERE $var.player > 0 RETURN v + """ + Then a SemanticError should be raised at runtime: `$var.player', not exist variable `var' + When executing query: + """ + MATCH (v:player) WITH {k: $^} AS x RETUR x.k.player.name + """ + Then a SyntaxError should be raised at runtime: syntax error near `} AS x R' + When executing query: + """ + MATCH (v:player) WITH {k: $^.player.age} AS x RETURN x.k + """ + Then a SemanticError should be raised at runtime: Expression $^.player.age is not allowed to use in cypher + When executing query: + """ + MATCH (v:player) WITH {k: $var} AS x RETUR x.k.player.name + """ + Then a SyntaxError should be raised at runtime: Direct output of variable is prohibited near `{k: $var}' Scenario: match with tag filter When executing query: From b391aad962452c3be4cae8c302c2d4083a8795b0 Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:04:49 +0800 Subject: [PATCH 34/51] Fix mistake push down limit with skip. (#5241) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- ...tEdgesTransformAppendVerticesLimitRule.cpp | 3 +- .../optimizer/rule/GetEdgesTransformRule.cpp | 3 +- tests/tck/features/match/Scan.feature | 123 ++++++++++++++++++ 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp index ae2421c811e..c6971129237 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp +++ b/src/graph/optimizer/rule/GetEdgesTransformAppendVerticesLimitRule.cpp @@ -121,7 +121,8 @@ StatusOr GetEdgesTransformAppendVerticesLimitRule::tra newLimitGroupNode->dependsOn(newAppendVerticesGroup); - auto *newScanEdges = GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->count(qctx)); + auto *newScanEdges = + GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->offset() + limit->count(qctx)); if (newScanEdges == nullptr) { return TransformResult::noTransform(); } diff --git a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp index 695ab2c4dfe..e72e4c213eb 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformRule.cpp +++ b/src/graph/optimizer/rule/GetEdgesTransformRule.cpp @@ -100,7 +100,8 @@ StatusOr GetEdgesTransformRule::transform( newProjectGroupNode->dependsOn(newLimitGroup); newProject->setInputVar(newLimit->outputVar()); - auto *newScanEdges = GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->count(qctx)); + auto *newScanEdges = + GetEdgesTransformUtils::traverseToScanEdges(traverse, limit->offset() + limit->count(qctx)); if (newScanEdges == nullptr) { return TransformResult::noTransform(); } diff --git a/tests/tck/features/match/Scan.feature b/tests/tck/features/match/Scan.feature index fc84d2f70f5..b2acd24964d 100644 --- a/tests/tck/features/match/Scan.feature +++ b/tests/tck/features/match/Scan.feature @@ -78,6 +78,67 @@ Feature: Match seek by scan | Name | | "Mary" | + Scenario: query vertices by scan with skip limit + When executing query: + """ + MATCH (v) + RETURN v.person.name AS name + SKIP 10 LIMIT 4 + """ + Then the result should be, in any order: + | name | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + When executing query: + """ + MATCH (v) + RETURN v.person.name AS name + SKIP 10 LIMIT 5 + """ + Then the result should be, in any order: + | name | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + When executing query: + """ + MATCH (v) + RETURN v.person.name AS name + SKIP 10 LIMIT 7 + """ + Then the result should be, in any order: + | name | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + When executing query: + """ + MATCH (v) + RETURN v.person.name AS name + SKIP 10 LIMIT 11 + """ + Then the result should be, in any order: + | name | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + Scenario: query vertices by scan failed When executing query: """ @@ -168,3 +229,65 @@ Feature: Match seek by scan LIMIT 3 """ Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. + + # #5223 + Scenario: query edge by scan with skip limit + When executing query: + """ + MATCH ()-[e]->() + RETURN type(e) AS Type + SKIP 10 LIMIT 4 + """ + Then the result should be, in any order: + | Type | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + When executing query: + """ + MATCH ()-[e]->() + RETURN type(e) AS Type + SKIP 10 LIMIT 5 + """ + Then the result should be, in any order: + | Type | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + When executing query: + """ + MATCH ()-[e]->() + RETURN type(e) AS Type + SKIP 10 LIMIT 7 + """ + Then the result should be, in any order: + | Type | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + When executing query: + """ + MATCH ()-[e]->() + RETURN type(e) AS Type + SKIP 10 LIMIT 11 + """ + Then the result should be, in any order: + | Type | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | + | /[\w_]+/ | From 19cc9b7ad4c46045f1579a858e71583791995f4d Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:32:48 +0800 Subject: [PATCH 35/51] fix delete fulltext index (#5239) * fix delete fulltext index * fix es delete error 1. remove get Rowreader if op is delete 2. delete es data when value is null Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- .../listener/elasticsearch/ESListener.cpp | 92 ++++++++++--------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/src/kvstore/listener/elasticsearch/ESListener.cpp b/src/kvstore/listener/elasticsearch/ESListener.cpp index 66927c8c5da..92435e6ec22 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.cpp +++ b/src/kvstore/listener/elasticsearch/ESListener.cpp @@ -73,6 +73,18 @@ void ESListener::pickTagAndEdgeData(BatchLogType type, const std::string& key, const std::string& value, const PickFunc& callback) { + bool isTag = nebula::NebulaKeyUtils::isTag(vIdLen_, key); + bool isEdge = nebula::NebulaKeyUtils::isEdge(vIdLen_, key); + if (!(isTag || isEdge)) { + return; + } + std::unordered_map ftIndexes; + nebula::RowReaderWrapper reader; + + std::string vid; + std::string src; + std::string dst; + int rank = 0; if (nebula::NebulaKeyUtils::isTag(vIdLen_, key)) { auto tagId = NebulaKeyUtils::getTagId(vIdLen_, key); auto ftIndexRes = schemaMan_->getFTIndex(spaceId_, tagId); @@ -80,34 +92,17 @@ void ESListener::pickTagAndEdgeData(BatchLogType type, LOG(ERROR) << ftIndexRes.status().message(); return; } - auto ftIndex = std::move(ftIndexRes).value(); - if (ftIndex.empty()) { - return; - } - auto reader = RowReaderWrapper::getTagPropReader(schemaMan_, spaceId_, tagId, value); - if (reader == nullptr) { - LOG(ERROR) << "get tag reader failed, tagID " << tagId; - return; - } - for (auto& index : ftIndex) { - if (index.second.get_fields().size() > 1) { - LOG(ERROR) << "Only one field will create fulltext index"; - } - auto field = index.second.get_fields().front(); - auto v = reader->getValueByName(field); - if (v.type() == Value::Type::NULLVALUE) { - continue; - } - if (v.type() != Value::Type::STRING) { - LOG(ERROR) << "Can't create fulltext index on type " << v.type(); + ftIndexes = std::move(ftIndexRes).value(); + if (type == BatchLogType::OP_BATCH_PUT) { + reader = RowReaderWrapper::getTagPropReader(schemaMan_, spaceId_, tagId, value); + if (reader == nullptr) { + LOG(ERROR) << "get tag reader failed, tagID " << tagId; + return; } - std::string indexName = index.first; - std::string vid = NebulaKeyUtils::getVertexId(vIdLen_, key).toString(); - vid = truncateVid(vid); - std::string text = std::move(v).getStr(); - callback(type, indexName, vid, "", "", 0, text); } - } else if (nebula::NebulaKeyUtils::isEdge(vIdLen_, key)) { + vid = NebulaKeyUtils::getVertexId(vIdLen_, key).toString(); + vid = truncateVid(vid); + } else { auto edgeType = NebulaKeyUtils::getEdgeType(vIdLen_, key); if (edgeType < 0) { return; @@ -116,33 +111,44 @@ void ESListener::pickTagAndEdgeData(BatchLogType type, if (!ftIndexRes.ok()) { return; } - auto ftIndex = std::move(ftIndexRes).value(); - auto reader = RowReaderWrapper::getEdgePropReader(schemaMan_, spaceId_, edgeType, value); - if (reader == nullptr) { - LOG(ERROR) << "get edge reader failed, schema ID " << edgeType; - return; - } - for (auto& index : ftIndex) { - if (index.second.get_fields().size() > 1) { - LOG(ERROR) << "Only one field will create fulltext index"; + ftIndexes = std::move(ftIndexRes).value(); + if (type == BatchLogType::OP_BATCH_PUT) { + reader = RowReaderWrapper::getEdgePropReader(schemaMan_, spaceId_, edgeType, value); + if (reader == nullptr) { + LOG(ERROR) << "get edge reader failed, schema ID " << edgeType; + return; } + } + src = NebulaKeyUtils::getSrcId(vIdLen_, key).toString(); + dst = NebulaKeyUtils::getDstId(vIdLen_, key).toString(); + rank = NebulaKeyUtils::getRank(vIdLen_, key); + + src = truncateVid(src); + dst = truncateVid(dst); + } + if (ftIndexes.empty()) { + return; + } + + for (auto& index : ftIndexes) { + if (index.second.get_fields().size() > 1) { + LOG(ERROR) << "Only one field will create fulltext index"; + } + std::string text; + std::string indexName = index.first; + if (type == BatchLogType::OP_BATCH_PUT) { auto field = index.second.get_fields().front(); auto v = reader->getValueByName(field); if (v.type() == Value::Type::NULLVALUE) { + callback(BatchLogType::OP_BATCH_REMOVE, indexName, vid, src, dst, 0, text); continue; } if (v.type() != Value::Type::STRING) { LOG(ERROR) << "Can't create fulltext index on type " << v.type(); } - std::string indexName = index.first; - std::string src = NebulaKeyUtils::getSrcId(vIdLen_, key).toString(); - std::string dst = NebulaKeyUtils::getDstId(vIdLen_, key).toString(); - int64_t rank = NebulaKeyUtils::getRank(vIdLen_, key); - std::string text = std::move(v).getStr(); - src = truncateVid(src); - dst = truncateVid(dst); - callback(type, indexName, "", src, dst, rank, text); + text = std::move(v).getStr(); } + callback(type, indexName, vid, src, dst, rank, text); } } From 87ebe358879687241e9caec2db49e9eb484b7a8e Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:44:08 +0800 Subject: [PATCH 36/51] Change the default value of session_reclaim_interval_secs to 60 seconds (#5246) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/graph/service/GraphFlags.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp index 7ffce40a526..5b754b1bd89 100644 --- a/src/graph/service/GraphFlags.cpp +++ b/src/graph/service/GraphFlags.cpp @@ -12,7 +12,7 @@ DEFINE_int32(client_idle_timeout_secs, 28800, "The number of seconds NebulaGraph service waits before closing the idle connections"); DEFINE_int32(session_idle_timeout_secs, 28800, "The number of seconds before idle sessions expire"); -DEFINE_int32(session_reclaim_interval_secs, 10, "Period we try to reclaim expired sessions"); +DEFINE_int32(session_reclaim_interval_secs, 60, "Period we try to reclaim expired sessions"); DEFINE_int32(num_netio_threads, 0, "The number of networking threads, 0 for number of physical CPU cores"); From c054a97be271fb6fa5e1c4937e47f11b1408fe37 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Fri, 13 Jan 2023 14:48:21 +0800 Subject: [PATCH 37/51] Enhance attribute-accessing expression to ensure self-consistency (#5230) * Revert "Remove all UNKNOWN_PROP as a type of null. (#4907)" (#5149) This reverts commit aa624162abaf92eeb6c2e5a8c35d31acc9366846. Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> * Enhance attribute-accessing expression to ensure self-consistency Fix tck Fix parser small delete Fix tck tck fmt fix ut fix ut Fix ut Fix tck Delete v.tag.prop check Fix tck Skip some tck cases related ngdata add test case Co-authored-by: Cheng Xuntao <7731943+xtcyclist@users.noreply.github.com> Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/common/datatypes/Value.cpp | 8 + src/common/datatypes/Value.h | 88 ++++--- src/common/expression/AttributeExpression.cpp | 34 ++- .../test/AttributeExpressionTest.cpp | 40 ++- .../test/ListComprehensionExpressionTest.cpp | 9 +- .../test/PredicateExpressionTest.cpp | 18 +- src/graph/validator/MatchValidator.cpp | 5 - .../tck/features/expression/Attribute.feature | 6 +- .../features/expression/Attribute1.feature | 1 - .../expression/ListComprehension.feature | 6 +- .../tck/features/expression/Predicate.feature | 8 +- tests/tck/features/expression/Reduce.feature | 12 +- tests/tck/features/match/Base.IntVid.feature | 114 +++++++- tests/tck/features/match/Base.feature | 248 +++++++++++++----- tests/tck/features/match/Path.feature | 8 + tests/tck/features/match/With.feature | 9 +- .../optimizer/PrunePropertiesRule.feature | 26 +- 17 files changed, 448 insertions(+), 192 deletions(-) diff --git a/src/common/datatypes/Value.cpp b/src/common/datatypes/Value.cpp index b0cd600effe..f17d9a4642b 100644 --- a/src/common/datatypes/Value.cpp +++ b/src/common/datatypes/Value.cpp @@ -292,6 +292,14 @@ Value::Value(Duration&& v) { setDU(std::make_unique(std::move(v))); } +Value::Value(const std::unordered_map& map) { + setM(std::make_unique(map)); +} + +Value::Value(std::unordered_map&& map) { + setM(std::make_unique(std::move(map))); +} + const std::string& Value::typeName() const { static const std::unordered_map typeNames = { {Type::__EMPTY__, "__EMPTY__"}, diff --git a/src/common/datatypes/Value.h b/src/common/datatypes/Value.h index 9749b529d3f..7047ccd0f53 100644 --- a/src/common/datatypes/Value.h +++ b/src/common/datatypes/Value.h @@ -91,49 +91,51 @@ struct Value { // matched ctor it will convert to bool type and the match the bool value // ctor, So we disable all pointer ctor except the char* template - Value(T*) = delete; // NOLINT - Value(const std::nullptr_t) = delete; // NOLINT - Value(const NullType& v); // NOLINT - Value(NullType&& v); // NOLINT - Value(const bool& v); // NOLINT - Value(bool&& v); // NOLINT - Value(const int8_t& v); // NOLINT - Value(int8_t&& v); // NOLINT - Value(const int16_t& v); // NOLINT - Value(int16_t&& v); // NOLINT - Value(const int32_t& v); // NOLINT - Value(int32_t&& v); // NOLINT - Value(const int64_t& v); // NOLINT - Value(int64_t&& v); // NOLINT - Value(const double& v); // NOLINT - Value(double&& v); // NOLINT - Value(const std::string& v); // NOLINT - Value(std::string&& v); // NOLINT - Value(const char* v); // NOLINT - Value(const Date& v); // NOLINT - Value(Date&& v); // NOLINT - Value(const Time& v); // NOLINT - Value(Time&& v); // NOLINT - Value(const DateTime& v); // NOLINT - Value(DateTime&& v); // NOLINT - Value(const Vertex& v); // NOLINT - Value(Vertex&& v); // NOLINT - Value(const Edge& v); // NOLINT - Value(Edge&& v); // NOLINT - Value(const Path& v); // NOLINT - Value(Path&& v); // NOLINT - Value(const List& v); // NOLINT - Value(List&& v); // NOLINT - Value(const Map& v); // NOLINT - Value(Map&& v); // NOLINT - Value(const Set& v); // NOLINT - Value(Set&& v); // NOLINT - Value(const DataSet& v); // NOLINT - Value(DataSet&& v); // NOLINT - Value(const Geography& v); // NOLINT - Value(Geography&& v); // NOLINT - Value(const Duration& v); // NOLINT - Value(Duration&& v); // NOLINT + Value(T*) = delete; // NOLINT + Value(const std::nullptr_t) = delete; // NOLINT + Value(const NullType& v); // NOLINT + Value(NullType&& v); // NOLINT + Value(const bool& v); // NOLINT + Value(bool&& v); // NOLINT + Value(const int8_t& v); // NOLINT + Value(int8_t&& v); // NOLINT + Value(const int16_t& v); // NOLINT + Value(int16_t&& v); // NOLINT + Value(const int32_t& v); // NOLINT + Value(int32_t&& v); // NOLINT + Value(const int64_t& v); // NOLINT + Value(int64_t&& v); // NOLINT + Value(const double& v); // NOLINT + Value(double&& v); // NOLINT + Value(const std::string& v); // NOLINT + Value(std::string&& v); // NOLINT + Value(const char* v); // NOLINT + Value(const Date& v); // NOLINT + Value(Date&& v); // NOLINT + Value(const Time& v); // NOLINT + Value(Time&& v); // NOLINT + Value(const DateTime& v); // NOLINT + Value(DateTime&& v); // NOLINT + Value(const Vertex& v); // NOLINT + Value(Vertex&& v); // NOLINT + Value(const Edge& v); // NOLINT + Value(Edge&& v); // NOLINT + Value(const Path& v); // NOLINT + Value(Path&& v); // NOLINT + Value(const List& v); // NOLINT + Value(List&& v); // NOLINT + Value(const Map& v); // NOLINT + Value(Map&& v); // NOLINT + Value(const Set& v); // NOLINT + Value(Set&& v); // NOLINT + Value(const DataSet& v); // NOLINT + Value(DataSet&& v); // NOLINT + Value(const Geography& v); // NOLINT + Value(Geography&& v); // NOLINT + Value(const Duration& v); // NOLINT + Value(Duration&& v); // NOLINT + Value(const std::unordered_map& map); // NOLINT + Value(std::unordered_map&& map); // NOLINT ~Value() { clear(); } diff --git a/src/common/expression/AttributeExpression.cpp b/src/common/expression/AttributeExpression.cpp index 20438134d57..81b58315999 100644 --- a/src/common/expression/AttributeExpression.cpp +++ b/src/common/expression/AttributeExpression.cpp @@ -17,23 +17,30 @@ const Value &AttributeExpression::eval(ExpressionContext &ctx) { auto &lvalue = left()->eval(ctx); auto &rvalue = right()->eval(ctx); DCHECK(rvalue.isStr()); + auto la = [&rvalue](const Tag &t) { return t.name == rvalue.getStr(); }; // TODO(dutor) Take care of the builtin properties, _src, _vid, _type, etc. switch (lvalue.type()) { - case Value::Type::MAP: - return lvalue.getMap().at(rvalue.getStr()); + case Value::Type::MAP: { + auto &m = lvalue.getMap().kvs; + auto iter = m.find(rvalue.getStr()); + if (iter == m.end()) { + return Value::kNullValue; + } + return iter->second; + } case Value::Type::VERTEX: { if (rvalue.getStr() == kVid) { result_ = lvalue.getVertex().vid; return result_; } - for (auto &tag : lvalue.getVertex().tags) { - auto iter = tag.props.find(rvalue.getStr()); - if (iter != tag.props.end()) { - return iter->second; - } + const auto &tags = lvalue.getVertex().tags; + auto iter = std::find_if(tags.begin(), tags.end(), la); + if (iter == tags.end()) { + return Value::kNullValue; } - return Value::kNullValue; + result_.setMap(Map(iter->props)); + return result_; } case Value::Type::EDGE: { DCHECK(!rvalue.getStr().empty()); @@ -64,11 +71,14 @@ const Value &AttributeExpression::eval(ExpressionContext &ctx) { case Value::Type::DATETIME: result_ = time::TimeUtils::getDateTimeAttr(lvalue.getDateTime(), rvalue.getStr()); return result_; - default: - if (lvalue.isNull() && lvalue.getNull() == NullType::UNKNOWN_PROP) { - // return UNKNOWN_PROP as plain null values, instead of bad type. - return Value::kNullValue; + case Value::Type::NULLVALUE: { + if (lvalue.isBadNull()) { + return Value::kNullBadType; } + return Value::kNullValue; + } + default: + // Unexpected data types return Value::kNullBadType; } } diff --git a/src/common/expression/test/AttributeExpressionTest.cpp b/src/common/expression/test/AttributeExpressionTest.cpp index 1258e953bfa..04fa594a150 100644 --- a/src/common/expression/test/AttributeExpressionTest.cpp +++ b/src/common/expression/test/AttributeExpressionTest.cpp @@ -82,6 +82,8 @@ TEST_F(AttributeExpressionTest, VertexAttribute) { Vertex vertex; vertex.vid = "vid"; vertex.tags.resize(2); + vertex.tags[0].name = "tag0"; + vertex.tags[1].name = "tag1"; vertex.tags[0].props = { {"Venus", "Mars"}, {"Mull", "Kintyre"}, @@ -92,28 +94,38 @@ TEST_F(AttributeExpressionTest, VertexAttribute) { {"Venus", "RocksShow"}, }; { - auto *left = ConstantExpression::make(&pool, Value(vertex)); - auto *right = LabelExpression::make(&pool, "Mull"); - auto expr = AttributeExpression::make(&pool, left, right); + auto expr = AttributeExpression::make( + &pool, + AttributeExpression::make(&pool, + ConstantExpression::make(&pool, Value(vertex)), + ConstantExpression::make(&pool, "tag0")), + ConstantExpression::make(&pool, "Mull")); + auto value = Expression::eval(expr, gExpCtxt); ASSERT_TRUE(value.isStr()); ASSERT_EQ("Kintyre", value.getStr()); } { - auto *left = ConstantExpression::make(&pool, Value(vertex)); - auto *right = LabelExpression::make(&pool, "Bip"); - auto expr = AttributeExpression::make(&pool, left, right); + auto expr = AttributeExpression::make( + &pool, + AttributeExpression::make(&pool, + ConstantExpression::make(&pool, Value(vertex)), + ConstantExpression::make(&pool, "tag1")), + ConstantExpression::make(&pool, "Bip")); auto value = Expression::eval(expr, gExpCtxt); ASSERT_TRUE(value.isStr()); ASSERT_EQ("Bop", value.getStr()); } { - auto *left = ConstantExpression::make(&pool, Value(vertex)); - auto *right = LabelExpression::make(&pool, "Venus"); - auto expr = AttributeExpression::make(&pool, left, right); + auto expr = AttributeExpression::make( + &pool, + AttributeExpression::make(&pool, + ConstantExpression::make(&pool, Value(vertex)), + ConstantExpression::make(&pool, "tag2")), + ConstantExpression::make(&pool, "Venus")); + auto value = Expression::eval(expr, gExpCtxt); - ASSERT_TRUE(value.isStr()); - ASSERT_EQ("Mars", value.getStr()); + ASSERT_TRUE(value.isNull()); } { auto *left = ConstantExpression::make(&pool, Value(vertex)); @@ -134,7 +146,7 @@ TEST_F(AttributeExpressionTest, DateTimeAttribute) { auto *right = LabelExpression::make(&pool, "not exist attribute"); auto expr = AttributeExpression::make(&pool, left, right); auto value = Expression::eval(expr, gExpCtxt); - ASSERT_EQ(Value::kNullValue, value); + ASSERT_EQ(Value::kNullUnknownProp, value); } { auto *left = ConstantExpression::make(&pool, Value(dt)); @@ -148,7 +160,7 @@ TEST_F(AttributeExpressionTest, DateTimeAttribute) { auto *right = LabelExpression::make(&pool, "not exist attribute"); auto expr = AttributeExpression::make(&pool, left, right); auto value = Expression::eval(expr, gExpCtxt); - ASSERT_EQ(Value::kNullValue, value); + ASSERT_EQ(Value::kNullUnknownProp, value); } { auto *left = ConstantExpression::make(&pool, Value(d)); @@ -162,7 +174,7 @@ TEST_F(AttributeExpressionTest, DateTimeAttribute) { auto *right = LabelExpression::make(&pool, "not exist attribute"); auto expr = AttributeExpression::make(&pool, left, right); auto value = Expression::eval(expr, gExpCtxt); - ASSERT_EQ(Value::kNullValue, value); + ASSERT_EQ(Value::kNullUnknownProp, value); } { auto *left = ConstantExpression::make(&pool, Value(t)); diff --git a/src/common/expression/test/ListComprehensionExpressionTest.cpp b/src/common/expression/test/ListComprehensionExpressionTest.cpp index 6e5931c0fbd..caffbc7869c 100644 --- a/src/common/expression/test/ListComprehensionExpressionTest.cpp +++ b/src/common/expression/test/ListComprehensionExpressionTest.cpp @@ -56,9 +56,12 @@ TEST_F(ListComprehensionExpressionTest, ListComprehensionEvaluate) { nullptr, ArithmeticExpression::makeAdd( &pool, - AttributeExpression::make(&pool, - VariableExpression::makeInner(&pool, "n"), - ConstantExpression::make(&pool, "age")), + AttributeExpression::make( + &pool, + AttributeExpression::make(&pool, + VariableExpression::makeInner(&pool, "n"), + ConstantExpression::make(&pool, "player")), + ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 5))); auto value = Expression::eval(expr, gExpCtxt); diff --git a/src/common/expression/test/PredicateExpressionTest.cpp b/src/common/expression/test/PredicateExpressionTest.cpp index 2fcc9c8f1b2..571281989de 100644 --- a/src/common/expression/test/PredicateExpressionTest.cpp +++ b/src/common/expression/test/PredicateExpressionTest.cpp @@ -50,9 +50,12 @@ TEST_F(PredicateExpressionTest, PredicateEvaluate) { FunctionCallExpression::make(&pool, "nodes", argList), RelationalExpression::makeGE( &pool, - AttributeExpression::make(&pool, - VariableExpression::makeInner(&pool, "n"), - ConstantExpression::make(&pool, "age")), + AttributeExpression::make( + &pool, + AttributeExpression::make(&pool, + VariableExpression::makeInner(&pool, "n"), + ConstantExpression::make(&pool, "player")), + ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 19))); auto value = Expression::eval(expr, gExpCtxt); @@ -100,9 +103,12 @@ TEST_F(PredicateExpressionTest, PredicateEvaluate) { FunctionCallExpression::make(&pool, "nodes", argList), RelationalExpression::makeGE( &pool, - AttributeExpression::make(&pool, - VariableExpression::makeInner(&pool, "n"), - ConstantExpression::make(&pool, "age")), + AttributeExpression::make( + &pool, + AttributeExpression::make(&pool, + VariableExpression::makeInner(&pool, "n"), + ConstantExpression::make(&pool, "player")), + ConstantExpression::make(&pool, "age")), ConstantExpression::make(&pool, 19))); auto value = Expression::eval(expr, gExpCtxt); diff --git a/src/graph/validator/MatchValidator.cpp b/src/graph/validator/MatchValidator.cpp index 18d6f09754f..492a36d76b5 100644 --- a/src/graph/validator/MatchValidator.cpp +++ b/src/graph/validator/MatchValidator.cpp @@ -982,11 +982,6 @@ Status MatchValidator::checkAlias( auto name = static_cast(refExpr)->left()->name(); auto res = getAliasType(aliasesAvailable, name); NG_RETURN_IF_ERROR(res); - if (res.value() == AliasType::kNode) { - return Status::SemanticError( - "To get the property of the vertex in `%s', should use the format `var.tag.prop'", - refExpr->toString().c_str()); - } return Status::OK(); } case Expression::Kind::kEdgeSrc: { diff --git a/tests/tck/features/expression/Attribute.feature b/tests/tck/features/expression/Attribute.feature index c471c3554c4..7bf039d4994 100644 --- a/tests/tck/features/expression/Attribute.feature +++ b/tests/tck/features/expression/Attribute.feature @@ -61,8 +61,8 @@ Feature: Attribute RETURN {k1 : 1, k2: true}.K1 AS k """ Then the result should be, in any order: - | k | - | UNKNOWN_PROP | + | k | + | NULL | When executing query: """ MATCH (v) WHERE id(v) == 'Tim Duncan' RETURN v.player.name @@ -122,7 +122,7 @@ Feature: Attribute """ Then the result should be, in any order: | not_exists_attr | - | UNKNOWN_PROP | + | NULL | When executing query: """ MATCH (v) WHERE id(v) == 'Tim Duncan' RETURN v.player.not_exists_attr diff --git a/tests/tck/features/expression/Attribute1.feature b/tests/tck/features/expression/Attribute1.feature index d153258ee17..92cfcf9387a 100644 --- a/tests/tck/features/expression/Attribute1.feature +++ b/tests/tck/features/expression/Attribute1.feature @@ -1,7 +1,6 @@ # Copyright (c) 2021 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License. -@skip Feature: Attribute using test Background: diff --git a/tests/tck/features/expression/ListComprehension.feature b/tests/tck/features/expression/ListComprehension.feature index bcaa62d42eb..a94bbd10e1d 100644 --- a/tests/tck/features/expression/ListComprehension.feature +++ b/tests/tck/features/expression/ListComprehension.feature @@ -53,8 +53,8 @@ Feature: ListComprehension When executing query: """ MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) - RETURN [n IN nodes(p) WHERE n.name - NOT STARTS WITH "LeBron" | n.age + 100] AS r + RETURN [n IN nodes(p) WHERE n.player.name + NOT STARTS WITH "LeBron" | n.player.age + 100] AS r """ Then the result should be, in any order: | r | @@ -67,7 +67,7 @@ Feature: ListComprehension When executing query: """ MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) - RETURN [n IN nodes(p) | n.age + 100] AS r + RETURN [n IN nodes(p) | n.player.age + 100] AS r """ Then the result should be, in any order: | r | diff --git a/tests/tck/features/expression/Predicate.feature b/tests/tck/features/expression/Predicate.feature index 3853c812d10..a059899f1d2 100644 --- a/tests/tck/features/expression/Predicate.feature +++ b/tests/tck/features/expression/Predicate.feature @@ -285,9 +285,9 @@ Feature: Predicate """ MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) RETURN - nodes(p)[0].name AS n1, - nodes(p)[1].name AS n2, - all(n IN nodes(p) WHERE n.name NOT STARTS WITH "D") AS b + nodes(p)[0].player.name AS n1, + nodes(p)[1].player.name AS n2, + all(n IN nodes(p) WHERE n.player.name NOT STARTS WITH "D") AS b """ Then the result should be, in any order: | n1 | n2 | b | @@ -300,7 +300,7 @@ Feature: Predicate When executing query: """ MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) - RETURN single(n IN nodes(p) WHERE n.age > 40) AS b + RETURN single(n IN nodes(p) WHERE n.player.age > 40) AS b """ Then the result should be, in any order: | b | diff --git a/tests/tck/features/expression/Reduce.feature b/tests/tck/features/expression/Reduce.feature index cae899f1544..7026fb26d9a 100644 --- a/tests/tck/features/expression/Reduce.feature +++ b/tests/tck/features/expression/Reduce.feature @@ -40,9 +40,9 @@ Feature: Reduce """ MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) RETURN - nodes(p)[0].age AS age1, - nodes(p)[1].age AS age2, - reduce(totalAge = 100, n IN nodes(p) | totalAge + n.age) AS r + nodes(p)[0].player.age AS age1, + nodes(p)[1].player.age AS age2, + reduce(totalAge = 100, n IN nodes(p) | totalAge + n.player.age) AS r """ Then the result should be, in any order: | age1 | age2 | r | @@ -55,9 +55,9 @@ Feature: Reduce When executing query: """ MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) - RETURN nodes(p)[0].age AS age1, - nodes(p)[1].age AS age2, - reduce(x = 10, n IN nodes(p) | n.age - x) AS x + RETURN nodes(p)[0].player.age AS age1, + nodes(p)[1].player.age AS age2, + reduce(x = 10, n IN nodes(p) | n.player.age - x) AS x """ Then the result should be, in any order: | age1 | age2 | x | diff --git a/tests/tck/features/match/Base.IntVid.feature b/tests/tck/features/match/Base.IntVid.feature index 098a3ff22c5..c045f105835 100644 --- a/tests/tck/features/match/Base.IntVid.feature +++ b/tests/tck/features/match/Base.IntVid.feature @@ -542,34 +542,126 @@ Feature: Basic match """ Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. - Scenario: can't get property of vertex + Scenario: Get property or tag from a vertex When executing query: """ - MATCH (v:player{name:"Tim Duncan"}) return v.name + MATCH (v:player{name:"Tim Duncan"}) RETURN v.name AS vname """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `v.name', should use the format `var.tag.prop' + Then the result should be, in any order: + | vname | + | NULL | + When executing query: + """ + MATCH (v:player{name:"Tim Duncan"}) RETURN v.player AS vtag + """ + Then the result should be, in any order: + | vtag | + | {name:"Tim Duncan"} | When executing query: """ - MATCH (v:player)-[]->(b) WHERE v.age > 30 return v.player.name + MATCH (v:player)-[]->(b) WHERE v.age > 30 RETURN v.player.name AS vname """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `v.age', should use the format `var.tag.prop' + Then the result should be, in any order: + | vname | When executing query: """ - MATCH (v:player)-[:like]->(b) WHERE v.player.age > 30 return v.player.name, b.age + MATCH (v:player)-[:like]->(b) WHERE v.player.age > 30 WITH v.player.name AS vname, b.age AS bage RETURN vname, bage """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `b.age', should use the format `var.tag.prop' + Then the result should be, in any order: + | vname | bage | + | "Rajon Rondo" | NULL | + | "Manu Ginobili" | NULL | + | "Tracy McGrady" | NULL | + | "Tracy McGrady" | NULL | + | "Tracy McGrady" | NULL | + | "Tim Duncan" | NULL | + | "Tim Duncan" | NULL | + | "Chris Paul" | NULL | + | "Chris Paul" | NULL | + | "Chris Paul" | NULL | + | "Rudy Gay" | NULL | + | "Paul Gasol" | NULL | + | "Paul Gasol" | NULL | + | "Boris Diaw" | NULL | + | "Boris Diaw" | NULL | + | "Aron Baynes" | NULL | + | "Carmelo Anthony" | NULL | + | "Carmelo Anthony" | NULL | + | "Carmelo Anthony" | NULL | + | "Danny Green" | NULL | + | "Danny Green" | NULL | + | "Danny Green" | NULL | + | "Vince Carter" | NULL | + | "Vince Carter" | NULL | + | "Jason Kidd" | NULL | + | "Jason Kidd" | NULL | + | "Jason Kidd" | NULL | + | "Marc Gasol" | NULL | + | "Grant Hill" | NULL | + | "Steve Nash" | NULL | + | "Steve Nash" | NULL | + | "Steve Nash" | NULL | + | "Steve Nash" | NULL | + | "Tony Parker" | NULL | + | "Tony Parker" | NULL | + | "Tony Parker" | NULL | + | "Marco Belinelli" | NULL | + | "Marco Belinelli" | NULL | + | "Marco Belinelli" | NULL | + | "Yao Ming" | NULL | + | "Yao Ming" | NULL | + | "Dirk Nowitzki" | NULL | + | "Dirk Nowitzki" | NULL | + | "Dirk Nowitzki" | NULL | + | "Shaquille O'Neal" | NULL | + | "Shaquille O'Neal" | NULL | + | "LaMarcus Aldridge" | NULL | + | "LaMarcus Aldridge" | NULL | + | "Tiago Splitter" | NULL | + | "Tiago Splitter" | NULL | + | "Ray Allen" | NULL | + | "LeBron James" | NULL | + | "Amar'e Stoudemire" | NULL | + | "Dwyane Wade" | NULL | + | "Dwyane Wade" | NULL | + | "Dwyane Wade" | NULL | + When executing query: + """ + MATCH (:player{name:"Tony Parker"})-[r]->(m) where exists(m.age) RETURN r + """ + Then the result should be, in any order: + | r | When executing query: """ - MATCH (:player{name:"Tony Parker"})-[r]->(m) where exists(m.age) return r + MATCH (:player{name:"Tony Parker"})-[r]->(m) where exists(m.player.age) RETURN r """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `m.age', should use the format `var.tag.prop' + Then the result should be, in any order, with relax comparison: + | r | + | [:teammate "Tony Parker"->"Kyle Anderson" @0 {end_year: 2016, start_year: 2014}] | + | [:teammate "Tony Parker"->"LaMarcus Aldridge" @0 {end_year: 2018, start_year: 2015}] | + | [:teammate "Tony Parker"->"Manu Ginobili" @0 {end_year: 2018, start_year: 2002}] | + | [:teammate "Tony Parker"->"Tim Duncan" @0 {end_year: 2016, start_year: 2001}] | + | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | When executing query: """ MATCH (v :player{name:"Tim Duncan"})-[]-(v2)-[]-(v3) WITH v3.name as names - RETURN count(names) + RETURN count(names) AS c + """ + Then the result should be, in any order: + | c | + | 0 | + When executing query: + """ + MATCH (v:player{name:"Tim Duncan"})-[]-(v2)-[]-(v3) + WITH v3.player.name as names + RETURN count(distinct names) AS c """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `v3.name', should use the format `var.tag.prop' + Then the result should be, in any order: + | c | + | 25 | Scenario: filter is not a valid expression When executing query: diff --git a/tests/tck/features/match/Base.feature b/tests/tck/features/match/Base.feature index 9355b90eb8a..3f93d4be2c6 100644 --- a/tests/tck/features/match/Base.feature +++ b/tests/tck/features/match/Base.feature @@ -49,7 +49,7 @@ Feature: Basic match | 38 | When executing query: """ - MATCH (v:player {age: 29}) return v.player.name AS Name + MATCH (v:player {age: 29}) RETURN v.player.name AS Name """ Then the result should be, in any order: | Name | @@ -59,7 +59,7 @@ Feature: Basic match | 'Dejounte Murray' | When executing query: """ - MATCH (v:player {age: 29}) WHERE v.player.name STARTS WITH "J" return v.player.name AS Name + MATCH (v:player {age: 29}) WHERE v.player.name STARTS WITH "J" RETURN v.player.name AS Name """ Then the result should be, in any order: | Name | @@ -67,7 +67,7 @@ Feature: Basic match | 'Jonathon Simmons' | When executing query: """ - MATCH (v:player) WHERE v.player.age >= 38 AND v.player.age < 45 return v.player.name AS Name, v.player.age AS Age + MATCH (v:player) WHERE v.player.age >= 38 AND v.player.age < 45 RETURN v.player.name AS Name, v.player.age AS Age """ Then the result should be, in any order: | Name | Age | @@ -111,12 +111,12 @@ Feature: Basic match | ("Yao Ming" :player{age: 38, name: "Yao Ming"}) | 138 | When executing query: """ - MATCH (v:player) where v.player.age > 9223372036854775807+1 return v + MATCH (v:player) where v.player.age > 9223372036854775807+1 RETURN v """ Then a SemanticError should be raised at runtime: result of (9223372036854775807+1) cannot be represented as an integer When executing query: """ - MATCH (v:player) where v.player.age > -9223372036854775808-1 return v + MATCH (v:player) where v.player.age > -9223372036854775808-1 RETURN v """ Then a SemanticError should be raised at runtime: result of (-9223372036854775808-1) cannot be represented as an integer @@ -604,21 +604,21 @@ Feature: Basic match Scenario: Return path When executing query: """ - MATCH p = (n:player{name:"Tony Parker"}) return p,n + MATCH p = (n:player{name:"Tony Parker"}) RETURN p,n """ Then the result should be, in any order, with relax comparison: | p | n | | <("Tony Parker")> | ("Tony Parker") | When executing query: """ - MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) return p, n.player.name, m.player.name + MATCH p = (n:player{name:"LeBron James"})-[:like]->(m) RETURN p, n.player.name, m.player.name """ Then the result should be, in any order, with relax comparison: | p | n.player.name | m.player.name | | <("LeBron James")-[:like@0]->("Ray Allen")> | "LeBron James" | "Ray Allen" | When executing query: """ - MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) return p, n.player.name, m.player.name + MATCH p = (n:player{name:"LeBron James"})<-[:like]-(m) RETURN p, n.player.name, m.player.name """ Then the result should be, in any order, with relax comparison: | p | n.player.name | m.player.name | @@ -630,7 +630,7 @@ Feature: Basic match | <("LeBron James")<-[:like@0]-("Kyrie Irving")> | "LeBron James" | "Kyrie Irving" | When executing query: """ - MATCH p = (n:player{name:"LeBron James"})-[:like]-(m) return p, n.player.name, m.player.name + MATCH p = (n:player{name:"LeBron James"})-[:like]-(m) RETURN p, n.player.name, m.player.name """ Then the result should be, in any order, with relax comparison: | p | n.player.name | m.player.name | @@ -643,7 +643,7 @@ Feature: Basic match | <("LeBron James")-[:like@0]->("Ray Allen")> | "LeBron James" | "Ray Allen" | When executing query: """ - MATCH p = (n:player{name:"LeBron James"})-[:like]->(m)-[:like]->(k) return p, n.player.name, m.player.name, k.player.name + MATCH p = (n:player{name:"LeBron James"})-[:like]->(m)-[:like]->(k) RETURN p, n.player.name, m.player.name, k.player.name """ Then the result should be, in any order, with relax comparison: | p | n.player.name | m.player.name | k.player.name | @@ -678,28 +678,28 @@ Feature: Basic match Then the result should be, in any order, with relax comparison: | p | - @skip - Scenario: Unsupported combination of some cypher clauses + Scenario: Combination of some cypher clauses When executing query: """ - MATCH (v:player) MATCH (t:team) RETURN v, t + UNWIND ["Tony Parker", "Tim Duncan", "Yao Ming"] AS a MATCH (v:player) WHERE v.player.name == a RETURN distinct a, v """ - Then a SemanticError should be raised at runtime: Match clause is not supported to be followed by other cypher clauses - When executing query: - """ - UNWIND ["Tony Parker", "Tim Duncan", "Yao Ming"] AS a MATCH (v:player) RETURN a, v - """ - Then a SemanticError should be raised at runtime: Match clause is not supported to be followed by other cypher clauses + Then the result should be, in any order, with relax comparison: + | a | v | + | "Yao Ming" | ("Yao Ming" :player{age: 38, name: "Yao Ming"}) | + | "Tim Duncan" | ("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"}) | + | "Tony Parker" | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | When executing query: """ - WITH "Tony Parker" AS a MATCH (v:player) RETURN a, v + WITH "Tony Parker" AS a MATCH (v:player) WHERE v.player.name == a RETURN a, v """ - Then a SemanticError should be raised at runtime: Match clause is not supported to be followed by other cypher clauses + Then the result should be, in any order, with relax comparison: + | a | v | + | "Tony Parker" | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | Scenario: exists When executing query: """ - match (:player{name:"Tony Parker"})-[r]->() where exists(r.likeness) return r, exists({a:12}.a) + match (:player{name:"Tony Parker"})-[r]->() where exists(r.likeness) RETURN r, exists({a:12}.a) """ Then the result should be, in any order, with relax comparison: | r | exists({a:12}.a) | @@ -708,13 +708,13 @@ Feature: Basic match | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | true | When executing query: """ - match (:player{name:"Tony Parker"})-[r]->(m) where exists(m.player.likeness) return r, exists({a:12}.a) + match (:player{name:"Tony Parker"})-[r]->(m) where exists(m.player.likeness) RETURN r, exists({a:12}.a) """ Then the result should be, in any order, with relax comparison: | r | exists({a:12}.a) | When executing query: """ - match (:player{name:"Tony Parker"})-[r]->(m) where exists({abc:123}.abc) return r + match (:player{name:"Tony Parker"})-[r]->(m) where exists({abc:123}.abc) RETURN r """ Then the result should be, in any order, with relax comparison: | r | @@ -729,13 +729,13 @@ Feature: Basic match | [:serve "Tony Parker"->"Spurs" @0 {end_year: 2018, start_year: 1999}] | When executing query: """ - match (:player{name:"Tony Parker"})-[r]->(m) where exists({abc:123}.ab) return r + match (:player{name:"Tony Parker"})-[r]->(m) where exists({abc:123}.ab) RETURN r """ Then the result should be, in any order, with relax comparison: | r | When executing query: """ - match (:player{name:"Tony Parker"})-[r]->(m) where exists(m.player.age) return r + match (:player{name:"Tony Parker"})-[r]->(m) where exists(m.player.age) RETURN r """ Then the result should be, in any order, with relax comparison: | r | @@ -750,28 +750,28 @@ Feature: Basic match Scenario: filter evaluable When executing query: """ - match (v:player{age: -1}) return v + match (v:player{age: -1}) RETURN v """ Then the result should be, in any order, with relax comparison: | v | | ("Null1" :player{age: -1, name: NULL}) | When executing query: """ - match (v:player{age: +20}) return v + match (v:player{age: +20}) RETURN v """ Then the result should be, in any order, with relax comparison: | v | | ("Luka Doncic" :player{age: 20, name: "Luka Doncic"}) | When executing query: """ - match (v:player{age: 1+19}) return v + match (v:player{age: 1+19}) RETURN v """ Then the result should be, in any order, with relax comparison: | v | | ("Luka Doncic" :player{age: 20, name: "Luka Doncic"}) | When executing query: """ - match (v:player)-[e:like{likeness:-1}]->() return e + match (v:player)-[e:like{likeness:-1}]->() RETURN e """ Then the result should be, in any order, with relax comparison: | e | @@ -779,7 +779,7 @@ Feature: Basic match | [:like "Rajon Rondo"->"Ray Allen" @0 {likeness: -1}] | When executing query: """ - match (v:player)-[e:like{likeness:40+50+5}]->() return e + match (v:player)-[e:like{likeness:40+50+5}]->() RETURN e """ Then the result should be, in any order, with relax comparison: | e | @@ -790,7 +790,7 @@ Feature: Basic match | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | When executing query: """ - match (v:player)-[e:like{likeness:4*20+5}]->() return e + match (v:player)-[e:like{likeness:4*20+5}]->() RETURN e """ Then the result should be, in any order, with relax comparison: | e | @@ -798,17 +798,17 @@ Feature: Basic match | [:like "Steve Nash"->"Jason Kidd"@0{likeness:85}] | When executing query: """ - match (v:player)-[e:like{likeness:"99"}]->() return e + match (v:player)-[e:like{likeness:"99"}]->() RETURN e """ Then the result should be, in any order, with relax comparison: | e | When executing query: """ - match (v:player{age:"24"-1}) return v + match (v:player{age:"24"-1}) RETURN v """ Then a SemanticError should be raised at runtime: Type error `("24"-1)' - Scenario: No return + Scenario: No RETURN When executing query: """ MATCH (v:player{name:"abc"}) @@ -816,14 +816,14 @@ Feature: Basic match Then a SyntaxError should be raised at runtime: syntax error near `)' When executing query: """ - MATCH (v:player) where v.player.name return v + MATCH (v:player) where v.player.name RETURN v """ Then a ExecutionError should be raised at runtime: Failed to evaluate condition: v.player.name. For boolean conditions, please write in their full forms like == or IS [NOT] NULL. Scenario: Unimplemented features When executing query: """ - MATCH (v) return v + MATCH (v) RETURN v """ Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. When executing query: @@ -844,12 +844,12 @@ Feature: Basic match Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. When executing query: """ - MATCH () -[]-> (v) return * + MATCH () -[]-> (v) RETURN * """ Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. When executing query: """ - MATCH () --> (v) --> () return * + MATCH () --> (v) --> () RETURN * """ Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. # The 0 step means node scan in fact, but p and t has no label or properties for index seek @@ -860,34 +860,152 @@ Feature: Basic match """ Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. - Scenario: can't get property of vertex + Scenario: Get property or tag from a vertex + When executing query: + """ + MATCH (v:player{name:"Tim Duncan"}) RETURN v.name AS vname + """ + Then the result should be, in any order: + | vname | + | NULL | When executing query: """ - MATCH (v:player{name:"Tim Duncan"}) return v.name + MATCH (v:player{name:"Tim Duncan"}) RETURN v.player AS vtag """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `v.name', should use the format `var.tag.prop' + Then the result should be, in any order: + | vtag | + | {name:"Tim Duncan"} | When executing query: """ - MATCH (v:player)-[]->(b) WHERE v.age > 30 return v.player.name + MATCH (v:player)-[]->(b) WHERE v.age > 30 RETURN v.player.name AS vname """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `v.age', should use the format `var.tag.prop' + Then the result should be, in any order: + | vname | When executing query: """ - MATCH (v:player)-[:like]->(b) WHERE v.player.age > 30 return v.player.name, b.age + MATCH (v:player)-[:like]->(b) WHERE v.player.age > 30 WITH v.player.name AS vname, b.age AS bage RETURN vname, bage """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `b.age', should use the format `var.tag.prop' + Then the result should be, in any order: + | vname | bage | + | "Rajon Rondo" | NULL | + | "Manu Ginobili" | NULL | + | "Tracy McGrady" | NULL | + | "Tracy McGrady" | NULL | + | "Tracy McGrady" | NULL | + | "Tim Duncan" | NULL | + | "Tim Duncan" | NULL | + | "Chris Paul" | NULL | + | "Chris Paul" | NULL | + | "Chris Paul" | NULL | + | "Rudy Gay" | NULL | + | "Paul Gasol" | NULL | + | "Paul Gasol" | NULL | + | "Boris Diaw" | NULL | + | "Boris Diaw" | NULL | + | "Aron Baynes" | NULL | + | "Carmelo Anthony" | NULL | + | "Carmelo Anthony" | NULL | + | "Carmelo Anthony" | NULL | + | "Danny Green" | NULL | + | "Danny Green" | NULL | + | "Danny Green" | NULL | + | "Vince Carter" | NULL | + | "Vince Carter" | NULL | + | "Jason Kidd" | NULL | + | "Jason Kidd" | NULL | + | "Jason Kidd" | NULL | + | "Marc Gasol" | NULL | + | "Grant Hill" | NULL | + | "Steve Nash" | NULL | + | "Steve Nash" | NULL | + | "Steve Nash" | NULL | + | "Steve Nash" | NULL | + | "Tony Parker" | NULL | + | "Tony Parker" | NULL | + | "Tony Parker" | NULL | + | "Marco Belinelli" | NULL | + | "Marco Belinelli" | NULL | + | "Marco Belinelli" | NULL | + | "Yao Ming" | NULL | + | "Yao Ming" | NULL | + | "Dirk Nowitzki" | NULL | + | "Dirk Nowitzki" | NULL | + | "Dirk Nowitzki" | NULL | + | "Shaquille O'Neal" | NULL | + | "Shaquille O'Neal" | NULL | + | "LaMarcus Aldridge" | NULL | + | "LaMarcus Aldridge" | NULL | + | "Tiago Splitter" | NULL | + | "Tiago Splitter" | NULL | + | "Ray Allen" | NULL | + | "LeBron James" | NULL | + | "Amar'e Stoudemire" | NULL | + | "Dwyane Wade" | NULL | + | "Dwyane Wade" | NULL | + | "Dwyane Wade" | NULL | + When executing query: + """ + MATCH (:player{name:"Tony Parker"})-[r]->(m) where exists(m.age) RETURN r + """ + Then the result should be, in any order: + | r | When executing query: """ - MATCH (:player{name:"Tony Parker"})-[r]->(m) where exists(m.age) return r + MATCH (:player{name:"Tony Parker"})-[r]->(m) where exists(m.player.age) RETURN r """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `m.age', should use the format `var.tag.prop' + Then the result should be, in any order, with relax comparison: + | r | + | [:teammate "Tony Parker"->"Kyle Anderson" @0 {end_year: 2016, start_year: 2014}] | + | [:teammate "Tony Parker"->"LaMarcus Aldridge" @0 {end_year: 2018, start_year: 2015}] | + | [:teammate "Tony Parker"->"Manu Ginobili" @0 {end_year: 2018, start_year: 2002}] | + | [:teammate "Tony Parker"->"Tim Duncan" @0 {end_year: 2016, start_year: 2001}] | + | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | When executing query: """ MATCH (v :player{name:"Tim Duncan"})-[]-(v2)-[]-(v3) WITH v3.name as names - RETURN count(names) + RETURN count(names) AS c + """ + Then the result should be, in any order: + | c | + | 0 | + When executing query: """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `v3.name', should use the format `var.tag.prop' + MATCH (v:player{name:"Tim Duncan"})-[]-(v2)-[]-(v3) + WITH v3.player.name as names + RETURN count(distinct names) AS c + """ + Then the result should be, in any order: + | c | + | 25 | + When executing query: + """ + MATCH (v:player{name:"Tim Duncan"})-[]-(v2)-[]-(v3) + WITH {a : v, b: v3} AS m + RETURN distinct m.a.player.age AS vage, m.b.team AS bteam + """ + Then the result should be, in any order, with relax comparison: + | vage | bteam | + | 42 | NULL | + | 42 | {name: "Suns"} | + | 42 | {name: "Magic"} | + | 42 | {name: "Spurs"} | + | 42 | {name: "Lakers"} | + | 42 | {name: "Heat"} | + | 42 | {name: "Celtics"} | + | 42 | {name: "Cavaliers"} | + | 42 | {name: "Hornets"} | + | 42 | {name: "Warriors"} | + | 42 | {name: "Raptors"} | + | 42 | {name: "Kings"} | + | 42 | {name: "Hawks"} | + | 42 | {name: "Bulls"} | + | 42 | {name: "76ers"} | + | 42 | {name: "Jazz"} | + | 42 | {name: "Trail Blazers"} | + | 42 | {name: "Pistons"} | Scenario: filter is not a valid expression When executing query: @@ -944,7 +1062,7 @@ Feature: Basic match Then the execution should be successful When executing query: """ - MATCH (v) WHERE id(v) in ["v1", "v2", "v3", "v4"] return id(v) limit 10; + MATCH (v) WHERE id(v) in ["v1", "v2", "v3", "v4"] RETURN id(v) limit 10; """ Then the result should be, in any order, with relax comparison: | id(v) | @@ -960,7 +1078,7 @@ Feature: Basic match And wait 5 seconds When executing query: """ - MATCH (v) WHERE id(v) in ["v1", "v2", "v3", "v4"] return id(v) limit 10; + MATCH (v) WHERE id(v) in ["v1", "v2", "v3", "v4"] RETURN id(v) limit 10; """ Then the result should be, in any order, with relax comparison: | id(v) | @@ -971,7 +1089,7 @@ Feature: Basic match match (v)-[e:like]->() where id(v) == "Tim Duncan" and rank(e) == 0 - return * + RETURN * """ Then the result should be, in any order: | v | e | @@ -982,7 +1100,7 @@ Feature: Basic match match (v)-[e:like]->() where id(v) == "Tim Duncan" and rank(e) != 0 - return * + RETURN * """ Then the result should be, in any order: | v | e | @@ -993,7 +1111,7 @@ Feature: Basic match match (v)-[e:like]->() where id(v) == "Tim Duncan" and rank(e) == 0 - return * + RETURN * """ Then the result should be, in any order, with relax comparison: | v | e | @@ -1003,7 +1121,7 @@ Feature: Basic match """ match ()-[e]->() where rank(e) == 0 - return rank(e) + RETURN rank(e) limit 3 """ Then the result should be, in any order, with relax comparison: @@ -1015,7 +1133,7 @@ Feature: Basic match """ match ()-[e]->() where rank(e) != 0 - return rank(e) + RETURN rank(e) limit 1000 """ Then the result should be, in any order, with relax comparison: @@ -1030,7 +1148,7 @@ Feature: Basic match """ match ()-[e]->() where abs(rank(e)) != 0 and e.start_year > 2010 - return rank(e) + RETURN rank(e) limit 1000 """ Then the result should be, in any order, with relax comparison: @@ -1043,7 +1161,7 @@ Feature: Basic match """ match ()-[e]->() where abs(rank(e)) == 1 and e.start_year == 2016 - return e + RETURN e limit 1000 """ Then the result should be, in any order, with relax comparison: @@ -1053,7 +1171,7 @@ Feature: Basic match """ match ()-[e]->() where src(e) != "jack" - return rank(e) + RETURN rank(e) limit 10000 """ Then an ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. @@ -1061,7 +1179,7 @@ Feature: Basic match """ match ()-[e]->() where src(e) != 0 or abs(rank(e)) != 0 - return rank(e) + RETURN rank(e) limit 10000 """ Then an ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. @@ -1069,7 +1187,7 @@ Feature: Basic match Scenario: match_with_wrong_syntax When executing query: """ - MATCH (v{name: "Tim Duncan"}) return v + MATCH (v{name: "Tim Duncan"}) RETURN v """ Then a SemanticError should be raised at runtime: `name:"Tim Duncan"': No tag found for property. When executing query: @@ -1131,20 +1249,20 @@ Feature: Basic match Scenario: match with tag filter When executing query: """ - MATCH (a:team)-[e*0..1]-(b) where id(a) == 'Tim Duncan' return b + MATCH (a:team)-[e*0..1]-(b) where id(a) == 'Tim Duncan' RETURN b """ Then the result should be, in any order, with relax comparison: | b | When executing query: """ - MATCH (a:team)-[e*0..0]-(b) where id(a) in ['Tim Duncan', 'Spurs'] return b + MATCH (a:team)-[e*0..0]-(b) where id(a) in ['Tim Duncan', 'Spurs'] RETURN b """ Then the result should be, in any order, with relax comparison: | b | | ('Spurs') | When executing query: """ - MATCH (a:team)-[e*0..1]-(b) where id(a) in ['Tim Duncan', 'Spurs'] return b + MATCH (a:team)-[e*0..1]-(b) where id(a) in ['Tim Duncan', 'Spurs'] RETURN b """ Then the result should be, in any order, with relax comparison: | b | diff --git a/tests/tck/features/match/Path.feature b/tests/tck/features/match/Path.feature index 9f31eca21e5..4e6230cb7e2 100644 --- a/tests/tck/features/match/Path.feature +++ b/tests/tck/features/match/Path.feature @@ -38,6 +38,8 @@ Feature: Matching paths | count(p) | count(p2) | | 966 | 966 | + # The correctness of the following test cases needs to be verified, mark it @skip for now + @skip Scenario: overlapping aliases variables When executing query: """ @@ -323,6 +325,8 @@ Feature: Matching paths | 1 | 1 | 1 | 96 | | 3 | 3 | 3 | 99 | + # The correctness of the following test cases needs to be verified, mark it @skip for now + @skip Scenario: symmetry paths When executing query: """ @@ -360,6 +364,8 @@ Feature: Matching paths | 2 | | 2 | + # The correctness of the following test cases needs to be verified, mark it @skip for now + @skip Scenario: src dst variables in paths When executing query: """ @@ -404,6 +410,8 @@ Feature: Matching paths | count(*) | | 190 | + # The correctness of the following test cases needs to be verified, mark it @skip for now + @skip Scenario: edgelist or single edge in paths When executing query: """ diff --git a/tests/tck/features/match/With.feature b/tests/tck/features/match/With.feature index 5dd6278a82f..ad9dd034334 100644 --- a/tests/tck/features/match/With.feature +++ b/tests/tck/features/match/With.feature @@ -94,8 +94,8 @@ Feature: With clause RETURN x.c """ Then the result should be, in any order: - | x.c | - | UNKNOWN_PROP | + | x.c | + | NULL | Scenario: match with return When executing query: @@ -232,7 +232,10 @@ Feature: With clause WITH dst AS b RETURN b.age AS age, b """ - Then a SemanticError should be raised at runtime: To get the property of the vertex in `b.age', should use the format `var.tag.prop' + Then the result should be, in any order, with relax comparison: + | age | b | + | NULL | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | + | NULL | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | @skip Scenario: with match return diff --git a/tests/tck/features/optimizer/PrunePropertiesRule.feature b/tests/tck/features/optimizer/PrunePropertiesRule.feature index c2f32a43b98..5c4473a7be1 100644 --- a/tests/tck/features/optimizer/PrunePropertiesRule.feature +++ b/tests/tck/features/optimizer/PrunePropertiesRule.feature @@ -756,9 +756,9 @@ Feature: Prune Properties rule """ Then the result should be, in order, with relax comparison: | properties(src_v).age | properties(e).degree | name | src_v.player.sex | e.start_year | dst_v.player.age | - | 41 | UNKNOWN_PROP | "Dejounte Murray" | "男" | 2022 | 29 | + | 41 | NULL | "Dejounte Murray" | "男" | 2022 | 29 | | 41 | 88 | "Spurs" | "男" | 2002 | NULL | - | 41 | UNKNOWN_PROP | "Tiago Splitter" | "男" | 2022 | 34 | + | 41 | NULL | "Tiago Splitter" | "男" | 2022 | 34 | When executing query: """ match (src_v:player{name:"Manu Ginobili"})-[e*2]-(dst_v) @@ -779,12 +779,12 @@ Feature: Prune Properties rule order by degree, name, age limit 5; """ Then the result should be, in order, with relax comparison: - | properties(src_v).sex | degree | name | age | e[1].start_year | dst_v.player.age | - | "男" | UNKNOWN_PROP | "Aron Baynes" | 41 | 2022 | 32 | - | "男" | UNKNOWN_PROP | "Blake Griffin" | 41 | 2022 | 30 | - | "男" | UNKNOWN_PROP | "Boris Diaw" | 41 | 2022 | 36 | - | "男" | UNKNOWN_PROP | "Carmelo Anthony" | 41 | 2022 | 34 | - | "男" | UNKNOWN_PROP | "Chris Paul" | 41 | 2022 | 33 | + | properties(src_v).sex | degree | name | age | e[1].start_year | dst_v.player.age | + | "男" | NULL | "Aron Baynes" | 41 | 2022 | 32 | + | "男" | NULL | "Blake Griffin" | 41 | 2022 | 30 | + | "男" | NULL | "Boris Diaw" | 41 | 2022 | 36 | + | "男" | NULL | "Carmelo Anthony" | 41 | 2022 | 34 | + | "男" | NULL | "Chris Paul" | 41 | 2022 | 33 | When executing query: """ match (v1)-->(v2)-->(v3) where id(v1)=="Manu Ginobili" @@ -860,11 +860,11 @@ Feature: Prune Properties rule """ Then the result should be, in order, with relax comparison: | properties(e).degree1 | properties(e).degree1 | e2.a | dst_v.p.name | dst_v.player.sex1 | properties(src_v).name2 | - | UNKNOWN_PROP | UNKNOWN_PROP | NULL | NULL | NULL | UNKNOWN_PROP | - | UNKNOWN_PROP | UNKNOWN_PROP | NULL | NULL | NULL | UNKNOWN_PROP | - | UNKNOWN_PROP | UNKNOWN_PROP | NULL | NULL | NULL | UNKNOWN_PROP | - | UNKNOWN_PROP | UNKNOWN_PROP | NULL | NULL | NULL | UNKNOWN_PROP | - | UNKNOWN_PROP | UNKNOWN_PROP | NULL | NULL | NULL | UNKNOWN_PROP | + | NULL | NULL | NULL | NULL | NULL | NULL | + | NULL | NULL | NULL | NULL | NULL | NULL | + | NULL | NULL | NULL | NULL | NULL | NULL | + | NULL | NULL | NULL | NULL | NULL | NULL | + | NULL | NULL | NULL | NULL | NULL | NULL | Then drop the used space Scenario: Project on not exist tag From c69103a37c3a024bd9b60ab5e60e538eab6a8725 Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Fri, 13 Jan 2023 15:39:57 +0800 Subject: [PATCH 38/51] fix ft index of fixed string (#5251) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- src/meta/processors/index/FTIndexProcessor.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/meta/processors/index/FTIndexProcessor.cpp b/src/meta/processors/index/FTIndexProcessor.cpp index d5df8d97268..278dee1e3ee 100644 --- a/src/meta/processors/index/FTIndexProcessor.cpp +++ b/src/meta/processors/index/FTIndexProcessor.cpp @@ -65,18 +65,6 @@ void CreateFTIndexProcessor::process(const cpp2::CreateFTIndexReq& req) { onFinished(); return; } - // if the data type is fixed_string, - // the data length must be less than MAX_INDEX_TYPE_LENGTH. - // else if the data type is string, - // will be truncated to MAX_INDEX_TYPE_LENGTH bytes when data insert. - if (targetCol->get_type().get_type() == nebula::cpp2::PropertyType::FIXED_STRING && - *targetCol->get_type().get_type_length() > MAX_INDEX_TYPE_LENGTH) { - LOG(INFO) << "Unsupported data length more than " << MAX_INDEX_TYPE_LENGTH - << " bytes : " << col << "(" << *targetCol->get_type().get_type_length() << ")"; - handleErrorCode(nebula::cpp2::ErrorCode::E_UNSUPPORTED); - onFinished(); - return; - } } // Check fulltext index exist. From 52a09d1b820406db07e0dbcb0844ed4eb593da15 Mon Sep 17 00:00:00 2001 From: jimingquan Date: Fri, 13 Jan 2023 18:23:45 +0800 Subject: [PATCH 39/51] Add tck test (#5253) * add allpath test * add shortest path test case * add subgraph test case * add go test case * add go test case --- tests/tck/features/go/GO.feature | 63 +++++++ tests/tck/features/path/AllPath.feature | 168 ++++++++++++++++++- tests/tck/features/path/ShortestPath.feature | 45 +++++ tests/tck/features/subgraph/subgraph.feature | 5 + 4 files changed, 273 insertions(+), 8 deletions(-) diff --git a/tests/tck/features/go/GO.feature b/tests/tck/features/go/GO.feature index 9948f3eb869..e39a5058b97 100644 --- a/tests/tck/features/go/GO.feature +++ b/tests/tck/features/go/GO.feature @@ -14,6 +14,25 @@ Feature: Go Sentence Then the result should be, in any order, with relax comparison: | serve._dst | | "Spurs" | + When executing query: + """ + GO FROM "Tim Duncan" OVER serve WHERE 1 > 0 YIELD serve._dst + """ + Then the result should be, in any order, with relax comparison: + | serve._dst | + | "Spurs" | + When executing query: + """ + GO FROM "Tim Duncan" OVER serve WHERE 1 < 0 YIELD serve._dst + """ + Then the result should be, in any order, with relax comparison: + | serve._dst | + When executing query: + """ + GO FROM "Tim Duncan" OVER serve WHERE 'Tim Duncan' > 0 YIELD serve._dst + """ + Then the result should be, in any order, with relax comparison: + | serve._dst | When executing query: """ GO FROM "Tim Duncan" OVER serve YIELD DISTINCT properties(edge) | YIELD COUNT(*) @@ -76,6 +95,17 @@ Feature: Go Sentence | "Boris Diaw" | 2008 | 2012 | "Hornets" | | "Boris Diaw" | 2012 | 2016 | "Spurs" | | "Boris Diaw" | 2016 | 2017 | "Jazz" | + When executing query: + """ + GO FROM "Boris Diaw" OVER serve YIELD $^.player.name, serve.start_year, serve.end_year, $$.team.name, 1, 1>2 + """ + Then the result should be, in any order, with relax comparison: + | $^.player.name | serve.start_year | serve.end_year | $$.team.name | 1 | (1>2) | + | "Boris Diaw" | 2003 | 2005 | "Hawks" | 1 | false | + | "Boris Diaw" | 2005 | 2008 | "Suns" | 1 | false | + | "Boris Diaw" | 2008 | 2012 | "Hornets" | 1 | false | + | "Boris Diaw" | 2012 | 2016 | "Spurs" | 1 | false | + | "Boris Diaw" | 2016 | 2017 | "Jazz" | 1 | false | When executing query: """ GO FROM "Rajon Rondo" OVER serve WHERE serve.start_year >= 2013 AND serve.end_year <= 2018 @@ -175,6 +205,12 @@ Feature: Go Sentence Then the result should be, in any order, with relax comparison: | $^.player.name | | "Tony Parker" | + When executing query: + """ + GO FROM 'Tony Parker' OVER like WHERE like._dst IN ['Tim Duncan', 'Danny Green'] AND 1 > 2 YIELD $^.player.name + """ + Then the result should be, in any order, with relax comparison: + | $^.player.name | Scenario: assignment simple When executing query: @@ -1213,6 +1249,11 @@ Feature: Go Sentence GO FROM 'Tim Duncan' OVER like where like.likeness YIELD like._dst """ Then a SemanticError should be raised at runtime: `like.likeness', expected boolean, but was `INT' + When executing query: + """ + GO FROM 'Tim Duncan' OVER like where ($^)-[:like]->($$) YIELD like._dst + """ + Then a SyntaxError should be raised at runtime: syntax error near `)-[:like' Scenario: contain When executing query: @@ -1615,6 +1656,28 @@ Feature: Go Sentence | name | | EMPTY | + Scenario: negative step + When executing query: + """ + GO -1 STEPS FROM 'Tim Duncan' OVER serve BIDIRECT YIELD serve._dst + """ + Then a SyntaxError should be raised at runtime: syntax error near `-1 STEPS' + When executing query: + """ + GO -1 TO 2 STEPS FROM 'Tim Duncan' OVER serve BIDIRECT YIELD serve._dst + """ + Then a SyntaxError should be raised at runtime: syntax error near `-1 TO 2 ' + When executing query: + """ + GO -1 TO -2 STEPS FROM 'Tim Duncan' OVER serve BIDIRECT YIELD serve._dst + """ + Then a SyntaxError should be raised at runtime: syntax error near `-1 TO -2' + When executing query: + """ + GO 1 TO -2 STEPS FROM 'Tim Duncan' OVER serve BIDIRECT YIELD serve._dst + """ + Then a SyntaxError should be raised at runtime: syntax error near `-2 STEPS' + Scenario: zero step When executing query: """ diff --git a/tests/tck/features/path/AllPath.feature b/tests/tck/features/path/AllPath.feature index 53efe544b1f..91afd8c9342 100644 --- a/tests/tck/features/path/AllPath.feature +++ b/tests/tck/features/path/AllPath.feature @@ -3,8 +3,30 @@ # This source code is licensed under Apache 2.0 License. Feature: All Path - Scenario: [1] ALL Path + Background: Given a graph with space named "nba" + + Scenario: ALL Path zero step + When executing query: + """ + FIND ALL PATH FROM "Tim Duncan" TO "Tim Duncan" OVER * UPTO 0 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + When executing query: + """ + FIND ALL PATH FROM "Tim Duncan" TO "Spurs", "Tony Parker" OVER * UPTO 0 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + When executing query: + """ + FIND ALL PATH FROM "Tim Duncan", "Tony Parker" TO "Tim Duncan" OVER * UPTO 0 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + + Scenario: ALL Path one TO one When executing query: """ FIND ALL PATH FROM "Tim Duncan" TO "Tim Duncan" OVER * UPTO 2 STEPS YIELD path as p @@ -30,6 +52,13 @@ Feature: All Path | <("Tim Duncan")-[:like]->("Tony Parker")> | | <("Tim Duncan")-[:like]->("Manu Ginobili")-[:like]->("Tim Duncan")-[:like]->("Tony Parker")> | | <("Tim Duncan")-[:like]->("Tony Parker")-[:like]->("LaMarcus Aldridge")-[:like]->("Tony Parker")> | + When executing query: + """ + FIND ALL PATH FROM "Tim Duncan" TO "Tony Parker" OVER like, noexist UPTO 3 STEPS YIELD path as p + """ + Then a SemanticError should be raised at runtime: noexist not found in space [nba]. + + Scenario: ALL Path one TO M When executing query: """ FIND ALL PATH FROM "Tim Duncan" TO "Tony Parker", "Manu Ginobili" OVER like UPTO 3 STEPS YIELD path as p @@ -69,8 +98,99 @@ Feature: All Path | <("Tim Duncan")-[:like]->("Tony Parker")-[:like]->("LaMarcus Aldridge")-[:serve]->("Spurs")> | | <("Tim Duncan")-[:like]->("Tony Parker")-[:like]->("Manu Ginobili")-[:serve]->("Spurs")> | + Scenario: ALL PATH M to one + When executing query: + """ + FIND ALL PATH FROM "Tony Parker","Spurs" TO "Tim Duncan" OVER like,serve UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Tony Parker")-[:like@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + When executing query: + """ + FIND ALL PATH FROM "noexist", "Tony Parker","Spurs" TO "Tim Duncan" OVER like,serve UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Tony Parker")-[:like@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Tony Parker")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + + Scenario: ALL PATH M to N + When executing query: + """ + FIND ALL PATH FROM "Yao Ming","Manu Ginobili" TO "Tim Duncan", "Larkers" OVER * UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Yao Ming")-[:like@0 {}]->("Shaquille O'Neal")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Danny Green")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Danny Green")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + When executing query: + """ + FIND ALL PATH FROM "Yao Ming","Manu Ginobili" TO "Tim Duncan", "Larkers", "noexist" OVER * UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Yao Ming")-[:like@0 {}]->("Shaquille O'Neal")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Danny Green")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Danny Green")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("LaMarcus Aldridge")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:like@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:like@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tony Parker")-[:teammate@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:like@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:like@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")> | + | <("Manu Ginobili")-[:teammate@0 {}]->("Tim Duncan")-[:teammate@0 {}]->("Manu Ginobili")-[:like@0 {}]->("Tim Duncan")> | + Scenario: [1] ALL Path Run Time Input - Given a graph with space named "nba" When executing query: """ GO FROM "Tim Duncan" over * YIELD like._dst AS src, serve._src AS dst | @@ -101,7 +221,6 @@ Feature: All Path | <("Tony Parker")-[:like]->("Tim Duncan")-[:like]->("Manu Ginobili")-[:like]->("Tim Duncan")> | Scenario: [1] ALL Path With Limit - Given a graph with space named "nba" When executing query: """ FIND ALL PATH FROM "Tim Duncan" TO "Tony Parker","Spurs" OVER like,serve UPTO 3 STEPS YIELD path as p | @@ -127,7 +246,6 @@ Feature: All Path | <("Tony Parker")-[:like]->("Manu Ginobili")-[:like]->("Tim Duncan")> | Scenario: [1] ALL PATH REVERSELY - Given a graph with space named "nba" When executing query: """ FIND ALL PATH FROM "Tim Duncan" TO "Nobody","Spur" OVER like REVERSELY UPTO 3 STEPS YIELD path as p @@ -164,7 +282,6 @@ Feature: All Path | <("Tim Duncan")<-[:like]-("Tony Parker")<-[:like]-("LaMarcus Aldridge")<-[:like]-("Tony Parker")> | Scenario: [2] ALL PATH BIDIRECT - Given a graph with space named "nba" When executing query: """ FIND ALL PATH FROM "Tim Duncan" TO "Tony Parker" OVER like BIDIRECT UPTO 3 STEPS YIELD path as p @@ -210,8 +327,25 @@ Feature: All Path | <("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:like@0 {likeness: 95}]->("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"})-[:like@0 {likeness: 90}]->("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:like@0 {likeness: 95}]->("Tony Parker" :player{age: 36, name: "Tony Parker"})> | | <("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:like@0 {likeness: 95}]->("Tony Parker" :player{age: 36, name: "Tony Parker"})-[:like@0 {likeness: 90}]->("LaMarcus Aldridge" :player{age: 33, name: "LaMarcus Aldridge"})-[:like@0 {likeness: 75}]->("Tony Parker" :player{age: 36, name: "Tony Parker"})> | - Scenario: ALL Path WITH FILTER - Given a graph with space named "nba" + Scenario: ALL Path constant filter + When executing query: + """ + FIND ALL PATH WITH PROP FROM "Tim Duncan" TO "Yao Ming" OVER * BIDIRECT + WHERE 1 > 2 UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + When executing query: + """ + FIND ALL PATH WITH PROP FROM "Tim Duncan" TO "Yao Ming" OVER * BIDIRECT + WHERE 1 < 2 UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"})<-[:like@0 {likeness: 80}]-("Shaquille O'Neal" :player{age: 47, name: "Shaquille O'Neal"})<-[:like@0 {likeness: 90}]-("Yao Ming" :player{age: 38, name: "Yao Ming"})> | + | <("Tim Duncan" :player{age: 42, name: "Tim Duncan"} :bachelor{name: "Tim Duncan", speciality: "psychology"})-[:serve@0 {end_year: 2016, start_year: 1997}]->("Spurs" :team{name: "Spurs"})<-[:serve@0 {end_year: 2013, start_year: 2013}]-("Tracy McGrady" :player{age: 39, name: "Tracy McGrady"})<-[:like@0 {likeness: 90}]-("Yao Ming" :player{age: 38, name: "Yao Ming"})> | + + Scenario: ALL Path edge filter When executing query: """ FIND ALL PATH WITH PROP FROM "Tim Duncan" TO "Yao Ming" OVER * BIDIRECT @@ -242,6 +376,25 @@ Feature: All Path | <("Yao Ming" :player{age: 38, name: "Yao Ming"})-[:like@0 {likeness: 90}]->("Shaquille O'Neal" :player{age: 47, name: "Shaquille O'Neal"})-[:like@0 {likeness: 80}]->("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"})-[:teammate@0 {end_year: 2016, start_year: 2010}]->("Danny Green" :player{age: 31, name: "Danny Green"})> | | <("Yao Ming" :player{age: 38, name: "Yao Ming"})-[:like@0 {likeness: 90}]->("Tracy McGrady" :player{age: 39, name: "Tracy McGrady"})-[:serve@0 {end_year: 2000, start_year: 1997}]->("Raptors" :team{name: "Raptors"})<-[:serve@0 {end_year: 2019, start_year: 2018}]-("Danny Green" :player{age: 31, name: "Danny Green"})> | | <("Yao Ming" :player{age: 38, name: "Yao Ming"})-[:like@0 {likeness: 90}]->("Tracy McGrady" :player{age: 39, name: "Tracy McGrady"})-[:serve@0 {end_year: 2013, start_year: 2013}]->("Spurs" :team{name: "Spurs"})<-[:serve@0 {end_year: 2018, start_year: 2010}]-("Danny Green" :player{age: 31, name: "Danny Green"})> | + When executing query: + """ + FIND ALL PATH WITH PROP FROM "Yao Ming" TO "Danny Green" OVER * BIDIRECT + WHERE like.likeness is EMPTY OR like.likeness >= 80 AND 1 > 2 UPTO 3 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + When executing query: + """ + GO FROM "Tim Duncan" over * YIELD like._dst AS src, serve._src AS dst, like.likeness AS val | + FIND ALL PATH FROM $-.src TO $-.dst OVER like WHERE like.likeness > $-.val UPTO 3 STEPS YIELD path as p + """ + Then a SemanticError should be raised at runtime: Not support `(like.likeness>$-.val)' in where sentence. + When executing query: + """ + $v = GO FROM "Tim Duncan" over * YIELD like._dst AS src, serve._src AS dst, like.likeness AS val; + FIND ALL PATH FROM $v.src TO $v.dst OVER like WHERE like.likeness > $v.val UPTO 3 STEPS YIELD path as p + """ + Then a SemanticError should be raised at runtime: Not support `(like.likeness>$v.val)' in where sentence. Scenario: Dangling edge Given an empty graph @@ -279,7 +432,6 @@ Feature: All Path Then drop the used space Scenario: ALL PATH YIELD PATH - Given a graph with space named "nba" When executing query: """ FIND ALL PATH WITH PROP FROM "Yao Ming" TO "Danny Green" OVER * BIDIRECT diff --git a/tests/tck/features/path/ShortestPath.feature b/tests/tck/features/path/ShortestPath.feature index 3c950029381..e08cf5e51ee 100644 --- a/tests/tck/features/path/ShortestPath.feature +++ b/tests/tck/features/path/ShortestPath.feature @@ -6,6 +6,25 @@ Feature: Shortest Path Background: Given a graph with space named "nba" + Scenario: Shortest Path zero step + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan" , "Yao Ming" TO "Tony Parker" OVER like UPTO 0 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan", "Yao Ming" TO "Tony Parker", "Spurs" OVER * UPTO 0 STEPS YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan", "Yao Ming" TO "Tony Parker", "Spurs" OVER * UPTO -1 STEPS YIELD path as p + """ + Then a SyntaxError should be raised at runtime: syntax error near `-1 STEPS' + Scenario: [1] SinglePair Shortest Path When executing query: """ @@ -14,6 +33,32 @@ Feature: Shortest Path Then the result should be, in any order, with relax comparison: | p | | <("Tim Duncan")-[:like]->("Tony Parker")> | + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan" TO "Tony Parker", "noexist" OVER like YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Tim Duncan")-[:like]->("Tony Parker")> | + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan", "noexist" TO "Tony Parker" OVER like YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Tim Duncan")-[:like]->("Tony Parker")> | + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan", "noexist" TO "Tony Parker", "noexist" OVER like YIELD path as p + """ + Then the result should be, in any order, with relax comparison: + | p | + | <("Tim Duncan")-[:like]->("Tony Parker")> | + When executing query: + """ + FIND SHORTEST PATH FROM "Tim Duncan", "noexist" TO "Tony Parker", "noexist" OVER noexistedge,like YIELD path as p + """ + Then a SemanticError should be raised at runtime: noexistedge not found in space [nba]. Scenario: [2] SinglePair Shortest Path When executing query: diff --git a/tests/tck/features/subgraph/subgraph.feature b/tests/tck/features/subgraph/subgraph.feature index 9169159d30f..f06c8e2dcd1 100644 --- a/tests/tck/features/subgraph/subgraph.feature +++ b/tests/tck/features/subgraph/subgraph.feature @@ -57,6 +57,11 @@ Feature: subgraph $a = GO FROM "Tim Duncan" OVER like YIELD like._dst AS id, like._src AS id; GET SUBGRAPH WITH PROP FROM $a.id YIELD vertices as nodes """ Then a SemanticError should be raised at runtime: Duplicate Column Name : `id' + When executing query: + """ + GET SUBGRAPH FROM 'Tim Duncan' OUT like, noexist YIELD vertices as v + """ + Then a ExecutionError should be raised at runtime: EdgeNotFound: EdgeName `noexist` Scenario: zero step When executing query: From 18cdda13698ea829c4dd7cdc65436551ae0eda58 Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Sat, 14 Jan 2023 09:01:16 +0800 Subject: [PATCH 40/51] Add more session tests (#5256) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- tests/job/test_session.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/job/test_session.py b/tests/job/test_session.py index 5c524c0b58b..2d36c8878bd 100644 --- a/tests/job/test_session.py +++ b/tests/job/test_session.py @@ -74,8 +74,8 @@ def cleanup(self): def test_sessions(self): # 1: test add session with right username try: - client_ok = self.client_pool.get_session('session_user', '123456') - assert client_ok is not None + user_session = self.client_pool.get_session('session_user', '123456') + assert user_session is not None assert True except Exception as e: assert False, e @@ -114,7 +114,7 @@ def test_sessions(self): assert session_id != 0 # 4: test get session info - resp = client_ok.execute('USE nba') + resp = user_session.execute('USE nba') self.check_resp_succeeded(resp) # wait for session sync. @@ -158,6 +158,16 @@ def test_sessions(self): self.check_resp_succeeded(resp) time.sleep(3) + # 6: test privilege + # show sessions with non-root user, only the root user can show all sessions + try: + non_root_session = self.client_pool.get_session('session_user', '123456') + assert non_root_session is not None + resp = non_root_session.execute('SHOW SESSIONS') + self.check_resp_failed(resp) + except Exception as e: + assert False, e + def test_the_same_id_to_different_graphd(self): conn1 = self.get_connection(self.addr_host1, self.addr_port1) @@ -431,15 +441,8 @@ def test_kill_session_multi_graph(self): self.check_resp_succeeded(ResultSet(resp, 0)) assert user1_session_num == len(ResultSet(resp, 0).rows()) + 1 - # execute query with the killed session - resp = conn2.execute( - session_id2, - 'SHOW HOSTS', - ) - # the session has not been synced to host2, so the query should succeed - self.check_resp_succeeded(ResultSet(resp, 0)) - # wait for the session to be synced (in test session_reclaim_interval_secs=2) + # and execute a query with the killed session time.sleep(4) resp = conn2.execute( session_id2, From 7c80edba5342b9c4e3cc5d8bb7c450dae273330e Mon Sep 17 00:00:00 2001 From: liwenhui-soul <38217397+liwenhui-soul@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:40:16 +0800 Subject: [PATCH 41/51] Revert "do not check term for leader info by default para" (#5266) This reverts commit 593bffcbdff0e0c6b32335165c0bce2137af50dd. --- src/meta/ActiveHostsMan.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/meta/ActiveHostsMan.cpp b/src/meta/ActiveHostsMan.cpp index 1dd1bb811a5..aa5aa707e9b 100644 --- a/src/meta/ActiveHostsMan.cpp +++ b/src/meta/ActiveHostsMan.cpp @@ -13,7 +13,6 @@ DECLARE_int32(heartbeat_interval_secs); DEFINE_int32(agent_heartbeat_interval_secs, 60, "Agent heartbeat interval in seconds"); DECLARE_uint32(expired_time_factor); -DEFINE_bool(check_term_for_leader_info, false, "if check term when update leader info"); namespace nebula { namespace meta { @@ -46,16 +45,14 @@ nebula::cpp2::ErrorCode ActiveHostsMan::updateHostInfo(kvstore::KVStore* kv, TermID term = -1; nebula::cpp2::ErrorCode code; for (auto i = 0U; i != leaderKeys.size(); ++i) { - if (FLAGS_check_term_for_leader_info) { - if (statusVec[i].ok()) { - std::tie(std::ignore, term, code) = MetaKeyUtils::parseLeaderValV3(vals[i]); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - LOG(INFO) << apache::thrift::util::enumNameSafe(code); - continue; - } - if (terms[i] <= term) { - continue; - } + if (statusVec[i].ok()) { + std::tie(std::ignore, term, code) = MetaKeyUtils::parseLeaderValV3(vals[i]); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + LOG(INFO) << apache::thrift::util::enumNameSafe(code); + continue; + } + if (terms[i] <= term) { + continue; } } // write directly if not exist, or update if has greater term From 9aac43451fd7f2093276c7ae6b94dc6c005d413f Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:15:59 +0800 Subject: [PATCH 42/51] modify ft index default limit size (#5260) * modify ft index default limit size * fix test Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> --- src/common/expression/TextSearchExpression.h | 2 +- src/graph/util/test/FTindexUtilsTest.cpp | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/common/expression/TextSearchExpression.h b/src/common/expression/TextSearchExpression.h index 078068111ae..b4239d91933 100644 --- a/src/common/expression/TextSearchExpression.h +++ b/src/common/expression/TextSearchExpression.h @@ -85,7 +85,7 @@ class TextSearchArgument final { std::string val_; std::string op_; int32_t fuzziness_{-2}; - int32_t limit_{-1}; + int32_t limit_{10000}; int32_t timeout_{-1}; }; diff --git a/src/graph/util/test/FTindexUtilsTest.cpp b/src/graph/util/test/FTindexUtilsTest.cpp index f131d054993..41dff6a0dd4 100644 --- a/src/graph/util/test/FTindexUtilsTest.cpp +++ b/src/graph/util/test/FTindexUtilsTest.cpp @@ -70,7 +70,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { esResult.items = items; { MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", -1, -1)) + EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", 10000, -1)) .WillOnce(Return(esResult)); auto argument = TextSearchArgument::make(&pool, tagName, propName, "prefix_pattern"); auto expr = TextSearchExpression::makePrefix(&pool, argument); @@ -84,7 +84,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { { plugin::ESQueryResult emptyEsResult; MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", -1, -1)) + EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", 10000, -1)) .WillOnce(Return(emptyEsResult)); auto argument = TextSearchArgument::make(&pool, tagName, propName, "prefix_pattern"); auto expr = TextSearchExpression::makePrefix(&pool, argument); @@ -95,7 +95,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { { Status status = Status::Error("mock error"); MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", -1, -1)) + EXPECT_CALL(mockESAdapter, prefix(indexName, "prefix_pattern", 10000, -1)) .Times(FLAGS_ft_request_retry_times) .WillRepeatedly(Return(status)); auto argument = TextSearchArgument::make(&pool, tagName, propName, "prefix_pattern"); @@ -106,7 +106,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { } { MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, wildcard(indexName, "wildcard_pattern", -1, -1)) + EXPECT_CALL(mockESAdapter, wildcard(indexName, "wildcard_pattern", 10000, -1)) .WillOnce(Return(esResult)); auto argument = TextSearchArgument::make(&pool, edgeName, propName, "wildcard_pattern"); auto expr = TextSearchExpression::makeWildcard(&pool, argument); @@ -121,7 +121,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { plugin::ESQueryResult singleEsResult; singleEsResult.items = {Item("a", "b", 1, "edge text")}; MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, wildcard(indexName, "wildcard_pattern", -1, -1)) + EXPECT_CALL(mockESAdapter, wildcard(indexName, "wildcard_pattern", 10000, -1)) .WillOnce(Return(singleEsResult)); auto argument = TextSearchArgument::make(&pool, edgeName, propName, "wildcard_pattern"); auto expr = TextSearchExpression::makeWildcard(&pool, argument); @@ -132,7 +132,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { } { MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, regexp(indexName, "regexp_pattern", -1, -1)) + EXPECT_CALL(mockESAdapter, regexp(indexName, "regexp_pattern", 10000, -1)) .WillOnce(Return(esResult)); auto argument = TextSearchArgument::make(&pool, edgeName, propName, "regexp_pattern"); auto expr = TextSearchExpression::makeRegexp(&pool, argument); @@ -145,7 +145,7 @@ TEST_F(FTIndexUtilsTest, rewriteTSFilter) { } { MockESAdapter mockESAdapter; - EXPECT_CALL(mockESAdapter, fuzzy(indexName, "fuzzy_pattern", "1", -1, -1)) + EXPECT_CALL(mockESAdapter, fuzzy(indexName, "fuzzy_pattern", "1", 10000, -1)) .WillOnce(Return(esResult)); auto argument = TextSearchArgument::make(&pool, tagName, propName, "fuzzy_pattern"); argument->setFuzziness(1); From 8a908fc50e9eb457eb4af13b54120dc13ba908f4 Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Tue, 17 Jan 2023 15:13:33 +0800 Subject: [PATCH 43/51] Test/yield (#5267) * Add some tests about yield. * Add more tests. Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- tests/tck/features/yield/yield.IntVid.feature | 149 ++++++++++++++++++ tests/tck/features/yield/yield.feature | 149 ++++++++++++++++++ 2 files changed, 298 insertions(+) diff --git a/tests/tck/features/yield/yield.IntVid.feature b/tests/tck/features/yield/yield.IntVid.feature index 0aef1b6b7e6..a6245d18a08 100644 --- a/tests/tck/features/yield/yield.IntVid.feature +++ b/tests/tck/features/yield/yield.IntVid.feature @@ -11,6 +11,141 @@ Feature: Yield Sentence Then the result should be, in any order: | 1 | | 1 | + When executing query: + """ + YIELD [1, 1.1, 1e2, 1.1e2, .3e4, 1.e4, 1234E-10, true] AS basic_value + """ + Then the result should be, in any order, with relax comparison: + | basic_value | + | [1, 1.1, 100.0, 110.0, 3000.0, 10000.0, 0.0000001234, true] | + When executing query: + """ + YIELD {p1: 1, p2: true, p3: "test"} AS r + """ + Then the result should be, in any order: + | r | + | {p1: 1, p2: true, p3: "test"} | + When executing query: + """ + YIELD true and false + """ + Then the result should be, in any order: + | (true AND false) | + | false | + When executing query: + """ + YIELD datetime('2012-03-04T22:11').year + """ + Then the result should be, in any order: + | datetime("2012-03-04T22:11").year | + | 2012 | + When executing query: + """ + YIELD datetime('2012-03-04T22:11').not_exists + """ + Then the result should be, in any order: + | datetime("2012-03-04T22:11").not_exists | + | UNKNOWN_PROP | + When executing query: + """ + YIELD CASE 1 WHEN 1 THEN 2 ELSE 3 END + """ + Then the result should be, in any order: + | CASE 1 WHEN 1 THEN 2 ELSE 3 END | + | 2 | + When executing query: + """ + YIELD abs(-1) + """ + Then the result should be, in any order: + | abs(-(1)) | + | 1 | + When executing query: + """ + YIELD v.l1.p1 + """ + Then a SemanticError should be raised at runtime: Invalid label identifiers: v + When executing query: + """ + YIELD l1 + """ + Then a SemanticError should be raised at runtime: Invalid label identifiers: l1 + When executing query: + """ + YIELD [i in [1, 2, 3] where i > 1 | i + 1] AS r + """ + Then the result should be, in any order: + | r | + | [3, 4] | + When executing query: + """ + YIELD all(n IN range(1, 5) WHERE n > 2) + """ + Then the result should be, in any order: + | all(n IN range(1,5) WHERE (n>2)) | + | false | + When executing query: + """ + YIELD like._dst + """ + Then a SemanticError should be raised at runtime: Only support input and variable in yield sentence. + When executing query: + """ + YIELD reduce(totalNum = 10, n IN range(1, 3) | totalNum + n) AS r + """ + Then the result should be, in any order: + | r | + | 16 | + When executing query: + """ + YIELD [1, 2, 3][2] + """ + Then the result should be, in any order: + | [1,2,3][2] | + | 3 | + When executing query: + """ + YIELD [1, 2, 3][0..1] + """ + Then the result should be, in any order: + | [1,2,3][0..1] | + | [1] | + When executing query: + """ + YIELD prefix(edge1.prop1,"高") + """ + Then a SyntaxError should be raised at runtime: syntax error near `(edge1.p' + When executing query: + """ + YIELD (INT)"1" + """ + Then the result should be, in any order: + | (INT)"1" | + | 1 | + When executing query: + """ + YIELD -(1) + """ + Then the result should be, in any order: + | -(1) | + | -1 | + When executing query: + """ + YIELD uuid() + """ + Then a SemanticError should be raised at runtime: Not supported expression `uuid()' for props deduction. + When executing query: + """ + YIELD $v + """ + Then a SyntaxError should be raised at runtime: Direct output of variable is prohibited near `$v' + When executing query: + """ + YIELD 1 AS num + """ + Then the result should be, in any order: + | num | + | 1 | When executing query: """ YIELD 1+1, (int)3.14, (string)(1+1), (string)true,"1+1" @@ -26,6 +161,20 @@ Feature: Yield Sentence | "Hello" | hash("Hello") | | "Hello" | 2275118702903107253 | + # TODO fix it + @skip + Scenario: Mistake + When executing query: + """ + YIELD count(*) + """ + Then a SemanticError should be raised at runtime: Don't support aggregate function without input. + When executing query: + """ + YIELD (v)-[:like]-() + """ + Then a SemanticError should be raised at runtime: Not support pattern expression. + Scenario: HashCall When executing query: """ diff --git a/tests/tck/features/yield/yield.feature b/tests/tck/features/yield/yield.feature index 47677b2cb3e..db9ad24b36c 100644 --- a/tests/tck/features/yield/yield.feature +++ b/tests/tck/features/yield/yield.feature @@ -4,6 +4,13 @@ Feature: Yield Sentence Given a graph with space named "nba" Scenario: Base + When executing query: + """ + YIELD 1 + """ + Then the result should be, in any order: + | 1 | + | 1 | When executing query: """ YIELD [1, 1.1, 1e2, 1.1e2, .3e4, 1.e4, 1234E-10, true] AS basic_value @@ -11,6 +18,134 @@ Feature: Yield Sentence Then the result should be, in any order, with relax comparison: | basic_value | | [1, 1.1, 100.0, 110.0, 3000.0, 10000.0, 0.0000001234, true] | + When executing query: + """ + YIELD {p1: 1, p2: true, p3: "test"} AS r + """ + Then the result should be, in any order: + | r | + | {p1: 1, p2: true, p3: "test"} | + When executing query: + """ + YIELD true and false + """ + Then the result should be, in any order: + | (true AND false) | + | false | + When executing query: + """ + YIELD datetime('2012-03-04T22:11').year + """ + Then the result should be, in any order: + | datetime("2012-03-04T22:11").year | + | 2012 | + When executing query: + """ + YIELD datetime('2012-03-04T22:11').not_exists + """ + Then the result should be, in any order: + | datetime("2012-03-04T22:11").not_exists | + | UNKNOWN_PROP | + When executing query: + """ + YIELD CASE 1 WHEN 1 THEN 2 ELSE 3 END + """ + Then the result should be, in any order: + | CASE 1 WHEN 1 THEN 2 ELSE 3 END | + | 2 | + When executing query: + """ + YIELD abs(-1) + """ + Then the result should be, in any order: + | abs(-(1)) | + | 1 | + When executing query: + """ + YIELD v.l1.p1 + """ + Then a SemanticError should be raised at runtime: Invalid label identifiers: v + When executing query: + """ + YIELD l1 + """ + Then a SemanticError should be raised at runtime: Invalid label identifiers: l1 + When executing query: + """ + YIELD [i in [1, 2, 3] where i > 1 | i + 1] AS r + """ + Then the result should be, in any order: + | r | + | [3, 4] | + When executing query: + """ + YIELD all(n IN range(1, 5) WHERE n > 2) + """ + Then the result should be, in any order: + | all(n IN range(1,5) WHERE (n>2)) | + | false | + When executing query: + """ + YIELD like._dst + """ + Then a SemanticError should be raised at runtime: Only support input and variable in yield sentence. + When executing query: + """ + YIELD reduce(totalNum = 10, n IN range(1, 3) | totalNum + n) AS r + """ + Then the result should be, in any order: + | r | + | 16 | + When executing query: + """ + YIELD [1, 2, 3][2] + """ + Then the result should be, in any order: + | [1,2,3][2] | + | 3 | + When executing query: + """ + YIELD [1, 2, 3][0..1] + """ + Then the result should be, in any order: + | [1,2,3][0..1] | + | [1] | + When executing query: + """ + YIELD prefix(edge1.prop1,"高") + """ + Then a SyntaxError should be raised at runtime: syntax error near `(edge1.p' + When executing query: + """ + YIELD (int)"1" + """ + Then the result should be, in any order: + | (INT)"1" | + | 1 | + When executing query: + """ + YIELD -(1) + """ + Then the result should be, in any order: + | -(1) | + | -1 | + When executing query: + """ + YIELD uuid() + """ + Then a SemanticError should be raised at runtime: Not supported expression `uuid()' for props deduction. + When executing query: + """ + YIELD $v + """ + Then a SyntaxError should be raised at runtime: Direct output of variable is prohibited near `$v' + When executing query: + """ + YIELD 1 AS num + """ + Then the result should be, in any order: + | num | + | 1 | When executing query: """ YIELD 1+1, (int)3.14, (string)(1+1), (string)true,"1+1" @@ -26,6 +161,20 @@ Feature: Yield Sentence | "Hello" | hash("Hello") | | "Hello" | 2275118702903107253 | + # TODO fix it + @skip + Scenario: Mistake + When executing query: + """ + YIELD count(*) + """ + Then a SemanticError should be raised at runtime: Don't support aggregate function without input. + When executing query: + """ + YIELD (v)-[:like]-() + """ + Then a SemanticError should be raised at runtime: Not support pattern expression. + Scenario: HashCall When executing query: """ From 6f972c9622ac2347614186e50d1ef55ff9df83b6 Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Tue, 17 Jan 2023 16:38:49 +0800 Subject: [PATCH 44/51] Add another cert to test CA don't match. (#5247) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- tests/cert/test.2.crt | 22 ++++++++++++++++++++++ tests/cert/test.2.csr | 18 ++++++++++++++++++ tests/cert/test.2.key | 30 ++++++++++++++++++++++++++++++ tests/cert/test.2.password | 1 + 4 files changed, 71 insertions(+) create mode 100644 tests/cert/test.2.crt create mode 100644 tests/cert/test.2.csr create mode 100644 tests/cert/test.2.key create mode 100644 tests/cert/test.2.password diff --git a/tests/cert/test.2.crt b/tests/cert/test.2.crt new file mode 100644 index 00000000000..b3771fbb003 --- /dev/null +++ b/tests/cert/test.2.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDszCCApsCFHDGB6747rJE8+MWP2IQdEfrONurMA0GCSqGSIb3DQEBCwUAMIGV +MQswCQYDVQQGEwJDTjERMA8GA1UECAwIWmhlamlhbmcxETAPBgNVBAcMCEhhbmd6 +aG91MRQwEgYDVQQKDAtWZXNvZnQgSW5jLjEMMAoGA1UECwwDRGV2MRMwEQYDVQQD +DApTaHlsb2NrIEhnMScwJQYJKoZIhvcNAQkBFhhzaHlsb2NrLmh1YW5nQHZlc29m +dC5jb20wHhcNMjMwMTEyMDgyMTA1WhcNMzMwMTA5MDgyMTA1WjCBlTELMAkGA1UE +BhMCQ04xETAPBgNVBAgMCFpoZWppYW5nMREwDwYDVQQHDAhIYW5nemhvdTEUMBIG +A1UECgwLVmVzb2Z0IEluYy4xDDAKBgNVBAsMA0RldjETMBEGA1UEAwwKU2h5bG9j +ayBIZzEnMCUGCSqGSIb3DQEJARYYc2h5bG9jay5odWFuZ0B2ZXNvZnQuY29tMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1WFQaXCffwmjwyUJhogVlHdZ +P1PkT/cY6uzQMfiMZibjdsxUrIeNcZU6CGGbS1a+s+BTwuqnOaIWre7qCRtHdzlk +ZaMqFW424WUPCgQxYAekIpw1N8H0gFoj3GqmV+580Ar9Y4L7vUwYwdzMPaFh7AgN +rWDMGOUFXTUW64st2IEIR6M6y0CwFvZMfbIgBrV1oyxfxQnb1OXQQS/Aq0MImstI +fCB+hNhefeGarvJI83O6y0wGXIohTUPKZgkRq+etIw1NzWS3bmCfAt3cwFkvkcMV +GW85OvD1wKCR5vPbTKTs46Nwqoj58+avqwzWvnzYADDRBZBV9/+94ljZ5RbDLwID +AQABMA0GCSqGSIb3DQEBCwUAA4IBAQCxAe7T3POLX16IgQrx416uXunpMBZVEmC/ +0cr/tX/ZVr1D94pZBDbNBplvrIYM0oGUAlzWuOnGhGONzKMTtsyf1JwG0/I/dg7/ +ng6uhiwDjUuB+dJ+CpoBtFNKr9u+Imw6ApDjcbvaN+ycwHwjBy1XX1SIN5bo+xfk +2K+dTAxSI1Zlsqs3spGVIc3IqdfUVnbYkNM3Nx3UYxIjgVfVW1WLf7BgsjvStZKF +pqixpST485YibWVxET3GKOJ4/fZwSl3DUGAMIVBZw4l1juZgG2IL5rFB2PPkNybk +0Xkw4r+RS92NK+CWtfjzQTLGdqxJpLVLt8Ed+q9aA/ugZqT8J3BX +-----END CERTIFICATE----- diff --git a/tests/cert/test.2.csr b/tests/cert/test.2.csr new file mode 100644 index 00000000000..b6d520c4e76 --- /dev/null +++ b/tests/cert/test.2.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC9zCCAd8CAQAwgZUxCzAJBgNVBAYTAkNOMREwDwYDVQQIDAhaaGVqaWFuZzER +MA8GA1UEBwwISGFuZ3pob3UxFDASBgNVBAoMC1Zlc29mdCBJbmMuMQwwCgYDVQQL +DANEZXYxEzARBgNVBAMMClNoeWxvY2sgSGcxJzAlBgkqhkiG9w0BCQEWGHNoeWxv +Y2suaHVhbmdAdmVzb2Z0LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBALgj6VJTBk7GlbaMW+lwyKjfW+ardnClLbNU+fh0gQGOAffRBz2RiTERzpva +3gpmG+JuxLOz4mUtcssGYkdX5p2Wpt3r2ne0h15UVth7sovyvmTQrS6opD+IZ9dm +7lbDxcZBnf+91gXmEB8f9pSc15eIwaUeqGH8s4YiPjVpM7/8VtRfhk5ps4RE2OiW +1k7vNhxEeBiW9bU5GmY/e8zN4YNJnSo3sXH47KlDekQeA4uZpJkM37GMEmvFImOP +11OvTIKEbPSuFDEvUKQMDB6xbTe83NUMV7tgMcTzoWbsPH/2GcSVTc32Kf1600tl +as6rl8gU2yrzjesEMOGxnDfFD1sCAwEAAaAcMBoGCSqGSIb3DQEJAjENDAtWZXNv +ZnQgSW5jLjANBgkqhkiG9w0BAQsFAAOCAQEASQzHAVdh3KqljVtxKtKvN2RyKhht +iMYS5YICC7ESLhcZhE4jp/XQ00jYs1tXRIczcmvstY/od9VmPX+ycL4OGUqzRaRF +Y4INCSy2/JIEz8aBoApuFzE5OZe/QS6ihwlZTSVW/Q/lTJeZ2N84Rj6ftlp4ZWLx +4NtyPuXWXIY/QOHp6rjbUYVcAvhFio2OIgfGFlV5E3rkmb/R2nU3vQ6BNK2sTpt7 +5ovXXbx7UrvS7/YVoP5+fg7iCSPOSA2t9yrD/N5htWSMiUJHp31EaTbNN8yxtVwb +ieAWLAuPrI0xoy6TLh2tDZX4HBw8ZwpBTDyi+tVdJdcsUzRM+w6xoTpjQg== +-----END CERTIFICATE REQUEST----- diff --git a/tests/cert/test.2.key b/tests/cert/test.2.key new file mode 100644 index 00000000000..d15589db759 --- /dev/null +++ b/tests/cert/test.2.key @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQINtWdG0nk//kCAggA +MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECER/BYBcGn8fBIIEyMtKPrObru3d +AS+3X4f0dSRErHSNUMOaiRe82RmklGX6Jrhn6C7C5nUA7poM+FBrbJYX6j9txcY8 +6MhskOXPtDYq4o7T0VQ0w9GIxS13nPEFz0zdRvimDKJPH5+rUhAv6DTM7YqZAf1D +oQmVPCpWRoYWOtA4NKYZFqv4kIUQQybPclr2vv7Q0pbKdqTvkyK46DRqozgLREj3 +iNRIU2xa5nMGrL8M+AjOKTjMJP5fbJXFr1ZezrPRdNg3EWaMAk+9zMfsluYsx6zZ +60rEHdCStYj4Gisc/FIGDf/CMGiuENrdO6gw5wsZU0MaqbgjUAbCMTngtrAciQiA +Q4bFcu33NOAhcnmiMpBFE3QMHBm1e0aAIR3sSqlJoIdxeD64b6D2+HtHQCvm8dsm ++nDDJVWimbieXQrki0GH/rHZzeDXTkMgMKvjRvpSb/Ksfio9IlPrNu88EMLmN9CC +cxwEPVxJheISyguZwMxlvz4sSsuxz3Msfujc5IJ4t6K4N2LtLqu4LB4WrTMZVGuG +56rHO2AbrEWKDRywwsm9x6gXppM/IMMKOddUlk02nKxd+oqeQQx2ZwbC2VeydAok +eXbhq8ccus2zZtLE/l6zhM0BCd6eJRp1rrTcx27bSf+KeE/GoYCabi3+ZI4xICW2 +pDPSbM0Hg0Ma09FRHYs/Fevmy3VfAZ8VR0zCVvCOku8avfIN5fZBRDaNmu1Fa2CP +WPTkpH3WBlsKyWO73aHVk1h94XI63hPY/5PIQJsgSCg7479nG57nFxjJQCOJzcN1 +SjkiXWFmOhLtBrAFc5YAypE+NDDWHKs5EsvUZZ4Xo4uSKtdJDISOZIHqpmp8Ntdp +z2ospT9CGwiVjjefmS6kgwRs+Ky6k6U89hOrdF08LSRM13OCeKQPUu6FuWmqlcBe +U0Pzh4Yxg+VVslbudIfLnlembY6lS5/jp1LFo3tH3wAFG64vdPlpaU16Xqc+rRCw +Ntdjt2v4Ju+S5WImLZO7DOMN93lyg6CYt/yLflMoZdKE7zIAon24U/RjrKn/z0KF +G8I022tll7ahrN7c28M9olzSqEgJMg8+tX1Bn6nxJZuLoTCd2AP47Dzw4rVpYFqh +HAQisNFAzlUntW9TUdMAVHEQY58/dAwZxEWXUMvgizn/3T6eRN4H1zw8BK+ae1ni +nGjlSfhvQJ7CwEhcSwCfYFSN72vkGv42g13CN1bik2aeajqbvS8H4plyHkTzQF44 +rqoLdqQTapm0JUU+h1OBApYI/ryM/VQiZr+6onXhlFzc+dl8bmDMpdswM4tWnU+T +BGAyet5DjKanLoz8rWvrm2F+feLbZeepfMK8pI+5mcfZk6B43xjcDYMZcXlt6Crl +3DyIsVzFGVN7nhtUk1VoQgCOO2aLpa9CjGM6Gb6ugsVqHF7z91XLLHLEc+/qVPWu +UJBXvvnj7H3HBAeOc5Uz1jUdNS+kPPVlQiyQBP++ggpThRL9whsgakSemUy7xtW2 +nzYJJhNxkFLEX1fMYucS74+ArI7MFnbuBLbeG/lfjkCl6ErOYCnaIsjK4+Z74Ros +odYQxGYMobvHWzxT12WYUGpbpDslTFqIuULRsRBxeGzF4Ud9e0YI2vnjB4RA5daH +G2KzjFNOtKdl1ng/1UoKIQ== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/tests/cert/test.2.password b/tests/cert/test.2.password new file mode 100644 index 00000000000..60b7570cd13 --- /dev/null +++ b/tests/cert/test.2.password @@ -0,0 +1 @@ +vesoft \ No newline at end of file From bcc7a83671589479476bb1b1d0bc7ec935b18b43 Mon Sep 17 00:00:00 2001 From: liwenhui-soul <38217397+liwenhui-soul@users.noreply.github.com> Date: Tue, 17 Jan 2023 18:48:10 +0800 Subject: [PATCH 45/51] fix baton miss reset in StorageJobExecutor (#5269) --- src/meta/processors/job/StorageJobExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/processors/job/StorageJobExecutor.cpp b/src/meta/processors/job/StorageJobExecutor.cpp index 1934c7aedec..1d179fa4e8b 100644 --- a/src/meta/processors/job/StorageJobExecutor.cpp +++ b/src/meta/processors/job/StorageJobExecutor.cpp @@ -231,7 +231,7 @@ folly::Future StorageJobExecutor::execute() { task.getErrorCode()); faildKV.emplace_back(std::move(taskKey), std::move(taskVal)); } - + baton.reset(); kvstore_->asyncMultiPut( kDefaultSpaceId, kDefaultPartId, std::move(faildKV), [&](nebula::cpp2::ErrorCode code) { if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { From cf5773ec23a0710e3ab960dfd24a5342d48bc354 Mon Sep 17 00:00:00 2001 From: Cheng Xuntao <7731943+xtcyclist@users.noreply.github.com> Date: Wed, 18 Jan 2023 17:03:08 +0800 Subject: [PATCH 46/51] Report errors on where clauses in optional match queries. (#5273) --- src/parser/parser.yy | 10 +- tests/tck/features/match/MatchById.feature | 2 +- .../match/MultiLineMultiQueryParts.feature | 92 +++++-------------- .../features/match/MultiQueryParts.feature | 42 +++++---- .../optimizer/PrunePropertiesRule.feature | 31 +++---- .../Read.feature | 6 +- .../interactive_workload/ComplexReads.feature | 3 +- 7 files changed, 74 insertions(+), 112 deletions(-) diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 966e81c6f77..a969625f581 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -1682,7 +1682,15 @@ match_clause $$ = new MatchClause($2, $3, false/*optional*/); } | KW_OPTIONAL KW_MATCH match_path_list where_clause { - $$ = new MatchClause($3, $4, true); + if ($4 != nullptr) { + SCOPE_EXIT { + delete $3; + delete $4; + }; + throw nebula::GraphParser::syntax_error(@4, "Where clause in optional match is not supported."); + } else { + $$ = new MatchClause($3, nullptr, true); + } } ; diff --git a/tests/tck/features/match/MatchById.feature b/tests/tck/features/match/MatchById.feature index c679c7d6486..f53e3ed01b4 100644 --- a/tests/tck/features/match/MatchById.feature +++ b/tests/tck/features/match/MatchById.feature @@ -1031,7 +1031,7 @@ Feature: Match By Id """ OPTIONAL MATCH (n) OPTIONAL MATCH (n) WHERE id(n) == 'James Harden' RETURN n """ - Then a ExecutionError should be raised at runtime: Scan vertices or edges need to specify a limit number, or limit number can not push down. + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE id(n) == 'James Harden'' When executing query: """ MATCH (v1)-[:like]->(v2:player)-[:serve]->(v3) diff --git a/tests/tck/features/match/MultiLineMultiQueryParts.feature b/tests/tck/features/match/MultiLineMultiQueryParts.feature index 7f1c72a40ca..1f0030c2831 100644 --- a/tests/tck/features/match/MultiLineMultiQueryParts.feature +++ b/tests/tck/features/match/MultiLineMultiQueryParts.feature @@ -197,103 +197,63 @@ Feature: Multi Line Multi Query Parts OPTIONAL MATCH (n)-[]->(l) WHERE id(n)=="Tony Parker" RETURN id(m) AS m, id(n) AS n, id(l) AS l; """ - Then the result should be, in any order: - | m | n | l | - | "Tim Duncan" | "Spurs" | NULL | - | "Tim Duncan" | "LaMarcus Aldridge" | NULL | - | "Tim Duncan" | "Tony Parker" | "Spurs" | - | "Tim Duncan" | "Tony Parker" | "LaMarcus Aldridge" | - | "Tim Duncan" | "Tony Parker" | "LaMarcus Aldridge" | - | "Tim Duncan" | "Tony Parker" | "Kyle Anderson" | - | "Tim Duncan" | "Tony Parker" | "Tim Duncan" | - | "Tim Duncan" | "Tony Parker" | "Tim Duncan" | - | "Tim Duncan" | "Tony Parker" | "Manu Ginobili" | - | "Tim Duncan" | "Tony Parker" | "Manu Ginobili" | - | "Tim Duncan" | "Tony Parker" | "Hornets" | - | "Tim Duncan" | "Tony Parker" | "Spurs" | - | "Tim Duncan" | "Tony Parker" | "LaMarcus Aldridge" | - | "Tim Duncan" | "Tony Parker" | "LaMarcus Aldridge" | - | "Tim Duncan" | "Tony Parker" | "Kyle Anderson" | - | "Tim Duncan" | "Tony Parker" | "Tim Duncan" | - | "Tim Duncan" | "Tony Parker" | "Tim Duncan" | - | "Tim Duncan" | "Tony Parker" | "Manu Ginobili" | - | "Tim Duncan" | "Tony Parker" | "Manu Ginobili" | - | "Tim Duncan" | "Tony Parker" | "Hornets" | - | "Tim Duncan" | "Danny Green" | NULL | - | "Tim Duncan" | "Manu Ginobili" | NULL | - | "Tim Duncan" | "Manu Ginobili" | NULL | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE id(n)=="Tony Parker"' When executing query: """ OPTIONAL match (v:player) WHERE v.player.age > 41 MATCH (v:player) WHERE v.player.age>40 RETURN count(*) AS count """ - Then the result should be, in order: - | count | - | 7 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 41' When executing query: """ - OPTIONAL match (v:player) WHERE v.player.age>43 + OPTIONAL match (v:player) WHERE v.player.age > 43 MATCH (n:player) WHERE n.player.age>40 RETURN count(*) AS count """ - Then the result should be, in order: - | count | - | 32 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 43' When executing query: """ - OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age<46 + OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age < 46 MATCH (v:player) WHERE v.player.age>43 RETURN count(*) AS count """ - Then the result should be, in any order: - | count | - | 2 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 40 and v.player.age < 46' When executing query: """ - MATCH (v:player) WHERE v.player.age > 40 and v.player.age<46 - OPTIONAL MATCH (v:player) WHERE v.player.age>43 + MATCH (v:player) WHERE v.player.age > 40 and v.player.age < 46 + OPTIONAL MATCH (v:player) WHERE v.player.age > 43 RETURN count(*) AS count """ - Then the result should be, in any order: - | count | - | 6 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 43' When executing query: """ OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age<46 OPTIONAL MATCH (v:player) WHERE v.player.age>43 RETURN count(*) AS count """ - Then the result should be, in any order: - | count | - | 6 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 40 and v.player.age<46' When executing query: """ - OPTIONAL MATCH (v:player) WHERE v.player.age>43 + OPTIONAL MATCH (v:player) WHERE v.player.age > 43 MATCH (v:player) WHERE v.player.age > 40 and v.player.age<46 RETURN count(*) AS count """ - Then the result should be, in any order: - | count | - | 2 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 43' When executing query: """ - MATCH (v:player) WHERE v.player.age>43 - OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age<46 + MATCH (v:player) WHERE v.player.age > 43 + OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age < 46 RETURN count(*) AS count """ - Then the result should be, in any order: - | count | - | 4 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 40 and v.player.age < 46' When executing query: """ - OPTIONAL MATCH (v:player) WHERE v.player.age>43 - OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age<46 + OPTIONAL MATCH (v:player) WHERE v.player.age > 43 + OPTIONAL MATCH (v:player) WHERE v.player.age > 40 and v.player.age < 46 RETURN count(*) AS count """ - Then the result should be, in any order: - | count | - | 4 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 43' # When the input of argument is NULL When executing query: """ @@ -370,22 +330,18 @@ Feature: Multi Line Multi Query Parts | 12 | When executing query: """ - OPTIONAL match (v:player) WHERE v.player.age>43 WITH v + OPTIONAL match (v:player) WHERE v.player.age > 43 WITH v MATCH (v:player) WHERE v.player.age>40 WITH v RETURN count(*) AS count """ - Then the result should be, in order: - | count | - | 4 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 43' When executing query: """ - OPTIONAL match (v:player) WHERE v.player.age>43 WITH v + OPTIONAL match (v:player) WHERE v.player.age > 43 WITH v MATCH (n:player) WHERE n.player.age>40 WITH v, n RETURN count(*) AS count """ - Then the result should be, in order: - | count | - | 32 | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age > 43' When executing query: """ MATCH (a:player{age:42}) WITH a @@ -411,10 +367,10 @@ Feature: Multi Line Multi Query Parts OPTIONAL MATCH (n)-->(v) WHERE v.player.age < m.player.age RETURN n,v """ - Then a SemanticError should be raised at runtime: The where clause of optional match statement that reference variables defined by other statements is not supported yet. + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE v.player.age < m.player.age' When executing query: """ MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan" OPTIONAL MATCH (n)-->(v) WHERE id(v) < id(m) RETURN count(*) AS count """ - Then a SemanticError should be raised at runtime: The where clause of optional match statement that reference variables defined by other statements is not supported yet. + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE id(v) < id(m)' diff --git a/tests/tck/features/match/MultiQueryParts.feature b/tests/tck/features/match/MultiQueryParts.feature index e722235b3a8..4388244ba48 100644 --- a/tests/tck/features/match/MultiQueryParts.feature +++ b/tests/tck/features/match/MultiQueryParts.feature @@ -177,36 +177,40 @@ Feature: Multi Query Parts When profiling query: """ MATCH (v1:player)-[:like*2..2]->(v2)-[e3:like]->(v4) where id(v1) == "Tony Parker" - OPTIONAL MATCH (v3:player)-[:like]->(v1)<-[e5]-(v4) where id(v3) == "Tim Duncan" return * + OPTIONAL MATCH (v3:player)-[:like]->(v1)<-[e5]-(v4) + with v1, v2, e3, v4, e5, v3 where id(v3) == "Tim Duncan" or id(v3) is NULL + return * """ Then the result should be, in any order, with relax comparison: - | v1 | v2 | e3 | v4 | v3 | e5 | - | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"LaMarcus Aldridge" @0] | ("LaMarcus Aldridge") | ("Tim Duncan") | [:like "LaMarcus Aldridge"->"Tony Parker" @0] | - | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Tony Parker" @0 ] | ("Tony Parker") | NULL | NULL | - | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Tony Parker" @0] | ("Tony Parker") | NULL | NULL | - | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"Tim Duncan" @0 ] | ("Tim Duncan") | ("Tim Duncan") | [:teammate "Tim Duncan"->"Tony Parker" @0] | - | ("Tony Parker") | ("Manu Ginobili") | [:like "Manu Ginobili"->"Tim Duncan" @0 ] | ("Tim Duncan") | ("Tim Duncan") | [:teammate "Tim Duncan"->"Tony Parker" @0] | - | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"Manu Ginobili" @0] | ("Manu Ginobili") | ("Tim Duncan") | [:teammate "Manu Ginobili"->"Tony Parker" @0] | - | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"Manu Ginobili" @0 ] | ("Manu Ginobili") | ("Tim Duncan") | [:teammate "Manu Ginobili"->"Tony Parker" @0] | - | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Manu Ginobili" @0 ] | ("Manu Ginobili") | ("Tim Duncan") | [:teammate "Manu Ginobili"->"Tony Parker" @0 ] | - | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Manu Ginobili" @0] | ("Manu Ginobili") | ("Tim Duncan") | [:teammate "Manu Ginobili"->"Tony Parker" @0] | + | v1 | v2 | e3 | v4 | e5 | v3 | + | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"LaMarcus Aldridge" @0] | ("LaMarcus Aldridge") | [:like "LaMarcus Aldridge"->"Tony Parker" @0] | ("Tim Duncan") | + | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Tony Parker" @0 ] | ("Tony Parker") | NULL | NULL | + | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Tony Parker" @0] | ("Tony Parker") | NULL | NULL | + | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"Tim Duncan" @0 ] | ("Tim Duncan") | [:teammate "Tim Duncan"->"Tony Parker" @0] | ("Tim Duncan") | + | ("Tony Parker") | ("Manu Ginobili") | [:like "Manu Ginobili"->"Tim Duncan" @0 ] | ("Tim Duncan") | [:teammate "Tim Duncan"->"Tony Parker" @0] | ("Tim Duncan") | + | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"Manu Ginobili" @0] | ("Manu Ginobili") | [:teammate "Manu Ginobili"->"Tony Parker" @0] | ("Tim Duncan") | + | ("Tony Parker") | ("Tony Parker") | [:like "Tony Parker"->"Manu Ginobili" @0 ] | ("Manu Ginobili") | [:teammate "Manu Ginobili"->"Tony Parker" @0] | ("Tim Duncan") | + | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Manu Ginobili" @0 ] | ("Manu Ginobili") | [:teammate "Manu Ginobili"->"Tony Parker" @0 ] | ("Tim Duncan") | + | ("Tony Parker") | ("Tim Duncan") | [:like "Tim Duncan"->"Manu Ginobili" @0] | ("Manu Ginobili") | [:teammate "Manu Ginobili"->"Tony Parker" @0] | ("Tim Duncan") | # The redudant Project after HashLeftJoin is removed now And the execution plan should be: | id | name | dependencies | profiling data | operator info | - | 19 | HashLeftJoin | 7,14 | | | - | 7 | Project | 6 | | | + | 22 | Project | 18 | | | + | 18 | Filter | 14 | | | + | 14 | HashLeftJoin | 7,13 | | | + | 7 | project | 6 | | | | 6 | AppendVertices | 5 | | | | 5 | Traverse | 20 | | | | 20 | Traverse | 2 | | | | 2 | Dedup | 1 | | | | 1 | PassThrough | 3 | | | | 3 | Start | | | | - | 14 | Project | 13 | | | - | 13 | Traverse | 21 | | | - | 21 | Traverse | 9 | | | - | 9 | Dedup | 8 | | | - | 8 | PassThrough | 10 | | | - | 10 | Start | | | | + | 13 | Project | 21 | | | + | 21 | AppendVertices | 11 | | | + | 11 | Traverse | 10 | | | + | 10 | AppendVertices | 9 | | | + | 9 | Traverse | 8 | | | + | 8 | Argument | | | | When executing query: """ MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan" diff --git a/tests/tck/features/optimizer/PrunePropertiesRule.feature b/tests/tck/features/optimizer/PrunePropertiesRule.feature index 5c4473a7be1..2b8193a8ebe 100644 --- a/tests/tck/features/optimizer/PrunePropertiesRule.feature +++ b/tests/tck/features/optimizer/PrunePropertiesRule.feature @@ -539,7 +539,6 @@ Feature: Prune Properties rule WHERE id(v) == 'Tim Duncan' AND b.player.age > 20 WITH v, count(b) AS countB, t OPTIONAL MATCH (v)-[:like]-()<-[:like]-(oldB)-[:serve]->(t) - WHERE oldB.player.age > 10 WITH v, countB, t, count(oldB) AS cb RETURN t.team.name, sum(countB) """ @@ -549,25 +548,23 @@ Feature: Prune Properties rule | "Hornets" | 3 | And the execution plan should be: | id | name | dependencies | operator info | - | 21 | Aggregate | 20 | | - | 20 | Aggregate | 19 | | - | 19 | HashLeftJoin | 10, 25 | | - | 10 | Aggregate | 23 | | - | 23 | Project | 22 | | - | 22 | Filter | 29 | | - | 29 | AppendVertices | 28 | { "props": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]" } | - | 28 | Traverse | 27 | {"vertexProps": "[{\"props\":[\"age\"] }]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | - | 27 | Traverse | 26 | {"vertexProps": "", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | - | 26 | Traverse | 2 | {"vertexProps": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}, { \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 19 | Aggregate | 18 | | + | 18 | Aggregate | 27 | | + | 27 | HashLeftJoin | 10,26 | | + | 10 | Aggregate | 21 | | + | 21 | Project | 20 | | + | 20 | Filter | 25 | | + | 25 | AppendVertices | 24 | { "props": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]" } | + | 24 | Traverse | 23 | {"vertexProps": "[{ \"props\":[\"age\"]}]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 23 | Traverse | 22 | {"vertexProps": "", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 22 | Traverse | 2 | {"vertexProps": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}, { \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | | 2 | Dedup | 1 | | | 1 | PassThrough | 3 | | | 3 | Start | | | - | 25 | Project | 24 | | - | 24 | Filter | 16 | | - | 16 | AppendVertices | 15 | { "props": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]"} | - | 15 | Traverse | 14 | {"vertexProps": "[{\"props\":[\"age\"] }]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | - | 14 | Traverse | 13 | {"vertexProps": "", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | - | 13 | Traverse | 11 | {"vertexProps": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}, { \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 26 | Project | 14 | | + | 14 | Traverse | 13 | {"vertexProps": "", "edgeProps": "[{ \"props\": [\"_src\", \"_type\", \"_rank\", \"_dst\", \"start_year\", \"end_year\"]}]" } | + | 13 | Traverse | 12 | {"vertexProps": "", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | + | 12 | Traverse | 11 | {"vertexProps": "[{ \"props\":[\"name\",\"age\",\"_tag\"]},{\"props\":[\"name\",\"speciality\",\"_tag\"] },{ \"props\":[\"name\",\"_tag\"]}]", "edgeProps": "[{ \"props\": [\"_type\", \"_rank\", \"_dst\"]}, { \"props\": [\"_type\", \"_rank\", \"_dst\"]}]" } | | 11 | Argument | | | @distonly diff --git a/tests/tck/ldbc/business_intelligence_workload/Read.feature b/tests/tck/ldbc/business_intelligence_workload/Read.feature index 523aa4d4788..176e00420f9 100644 --- a/tests/tck/ldbc/business_intelligence_workload/Read.feature +++ b/tests/tck/ldbc/business_intelligence_workload/Read.feature @@ -115,8 +115,7 @@ Feature: LDBC Business Intelligence Workload - Read tagName ASC LIMIT 100 """ - Then the result should be, in any order: - | tagName | countMonth1 | countMonth2 | diff | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE datetime('2012-06-01') <= message1.Message.creationDate AND message1.Messa' Scenario: 4. Popular topics in a country When executing query: @@ -530,8 +529,7 @@ Feature: LDBC Business Intelligence Workload - Read personCount DESC, messageCount DESC """ - Then the result should be, in order: - | messageCount | personCount | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE message.Message.content IS NOT NULL AND message.Message.length < 20 AND me' Scenario: 19. Stranger’s interaction # NOTICE: A big rewritten, have to test the correctness diff --git a/tests/tck/ldbc/interactive_workload/ComplexReads.feature b/tests/tck/ldbc/interactive_workload/ComplexReads.feature index dc7321b1be1..a0b470ebc59 100644 --- a/tests/tck/ldbc/interactive_workload/ComplexReads.feature +++ b/tests/tck/ldbc/interactive_workload/ComplexReads.feature @@ -137,8 +137,7 @@ Feature: LDBC Interactive Workload - Complex Reads sum(postsOnTag) AS postCount ORDER BY postCount DESC, tagName ASC """ - Then the result should be, in any order: - | tagName | postCount | + Then a SyntaxError should be raised at runtime: Where clause in optional match is not supported. near `WHERE oldPost.Post.creationDate < $startDate' Scenario: 5. New groups # {"personId":"6597069766734","minDate":"1288612800000"} From a94c6a9bf229672cee64cba4cca6d42ce58f3ec7 Mon Sep 17 00:00:00 2001 From: Yichen Wang <18348405+Aiee@users.noreply.github.com> Date: Thu, 19 Jan 2023 09:03:52 +0800 Subject: [PATCH 47/51] [test case] Check DML cases (#5264) * Check DML cases * Add chinses char tests Add more tests Add mero delete edge tests * Revert cases --- .../features/delete/DeleteEdge.IntVid.feature | 72 +++++++++++++++++++ tests/tck/features/delete/DeleteEdge.feature | 72 +++++++++++++++++++ .../features/delete/DeleteTag.IntVid.feature | 1 + tests/tck/features/delete/DeleteTag.feature | 3 +- .../tck/features/delete/DeleteVertex.feature | 2 +- .../tck/features/insert/Insert.IntVid.feature | 40 +++++++++++ tests/tck/features/insert/Insert.feature | 59 ++++++++++++++- .../insert/InsertEdgeOnDiffParts.feature | 2 +- 8 files changed, 247 insertions(+), 4 deletions(-) diff --git a/tests/tck/features/delete/DeleteEdge.IntVid.feature b/tests/tck/features/delete/DeleteEdge.IntVid.feature index 87794b5ded5..f676ea0ef85 100644 --- a/tests/tck/features/delete/DeleteEdge.IntVid.feature +++ b/tests/tck/features/delete/DeleteEdge.IntVid.feature @@ -135,6 +135,78 @@ Feature: Delete int vid of edge | "Zhangsan" | 50 | "Jack" | Then drop the used space + Scenario: delete edges delete the edge with rank 0 by default + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | int | + And having executed: + """ + CREATE TAG IF NOT EXISTS person(name string, age int); + CREATE EDGE IF NOT EXISTS friend(intimacy int); + """ + And having executed: + """ + INSERT VERTEX + person(name, age) + VALUES + hash("Zhangsan"):("Zhangsan", 22), + hash("Lisi"):("Lisi", 23); + INSERT EDGE + friend(intimacy) + VALUES + hash("Zhangsan")->hash("Lisi"):(1), + hash("Zhangsan")->hash("Lisi")@15:(2), + hash("Zhangsan")->hash("Lisi")@25:(3), + hash("Zhangsan")->hash("Lisi")@35:(4); + """ + # before delete get result by go + When executing query: + """ + GO FROM hash("Zhangsan") OVER friend + YIELD $^.person.name, friend.intimacy, friend._rank, friend._dst + """ + Then the result should be, in any order: + | $^.person.name | friend.intimacy | friend._rank | friend._dst | + | "Zhangsan" | 1 | 0 | hash("Lisi") | + | "Zhangsan" | 2 | 15 | hash("Lisi") | + | "Zhangsan" | 3 | 25 | hash("Lisi") | + | "Zhangsan" | 4 | 35 | hash("Lisi") | + # delete edge friend, by default only the edge with rank of 0 will be deleted + When executing query: + """ + DELETE EDGE friend hash("Zhangsan")->hash("Lisi"); + """ + Then the execution should be successful + # check result + When executing query: + """ + GO FROM hash("Zhangsan") OVER friend + YIELD $^.person.name, friend.intimacy, friend._rank, friend._dst + """ + Then the result should be, in any order: + | $^.person.name | friend.intimacy | friend._rank | friend._dst | + | "Zhangsan" | 2 | 15 | hash("Lisi") | + | "Zhangsan" | 3 | 25 | hash("Lisi") | + | "Zhangsan" | 4 | 35 | hash("Lisi") | + # delete all edges with different ranks + When executing query: + """ + GO FROM hash("Zhangsan") OVER friend YIELD id($^) AS src, friend._rank AS rank, friend._dst AS dst + | DELETE EDGE friend $-.src -> $-.dst @ $-.rank; + """ + Then the execution should be successful + # check result + When executing query: + """ + GO FROM hash("Zhangsan") OVER friend + YIELD $^.person.name, friend.intimacy, friend._rank, friend._dst + """ + Then the result should be, in any order: + | $^.person.name | friend.intimacy | friend._rank | friend._dst | + Then drop the used space + Scenario: delete edges use pipe Given load "nba_int_vid" csv data to a new space # test delete with pipe wrong vid type diff --git a/tests/tck/features/delete/DeleteEdge.feature b/tests/tck/features/delete/DeleteEdge.feature index abf06a7a343..adb9a5c755c 100644 --- a/tests/tck/features/delete/DeleteEdge.feature +++ b/tests/tck/features/delete/DeleteEdge.feature @@ -135,6 +135,78 @@ Feature: Delete string vid of edge | "Zhangsan" | 50 | "Jack" | Then drop the used space + Scenario: delete edges delete the edge with rank 0 by default + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(20) | + And having executed: + """ + CREATE TAG IF NOT EXISTS person(name string, age int); + CREATE EDGE IF NOT EXISTS friend(intimacy int); + """ + And having executed: + """ + INSERT VERTEX + person(name, age) + VALUES + "Zhangsan":("Zhangsan", 22), + "Lisi":("Lisi", 23); + INSERT EDGE + friend(intimacy) + VALUES + "Zhangsan"->"Lisi":(1), + "Zhangsan"->"Lisi"@15:(2), + "Zhangsan"->"Lisi"@25:(3), + "Zhangsan"->"Lisi"@35:(4); + """ + # before delete get result by go + When executing query: + """ + GO FROM "Zhangsan" OVER friend + YIELD $^.person.name, friend.intimacy, friend._rank, friend._dst + """ + Then the result should be, in any order: + | $^.person.name | friend.intimacy | friend._rank | friend._dst | + | "Zhangsan" | 1 | 0 | "Lisi" | + | "Zhangsan" | 2 | 15 | "Lisi" | + | "Zhangsan" | 3 | 25 | "Lisi" | + | "Zhangsan" | 4 | 35 | "Lisi" | + # delete edge friend, by default only the edge with rank of 0 will be deleted + When executing query: + """ + DELETE EDGE friend "Zhangsan"->"Lisi"; + """ + Then the execution should be successful + # check result + When executing query: + """ + GO FROM "Zhangsan" OVER friend + YIELD $^.person.name, friend.intimacy, friend._rank, friend._dst + """ + Then the result should be, in any order: + | $^.person.name | friend.intimacy | friend._rank | friend._dst | + | "Zhangsan" | 2 | 15 | "Lisi" | + | "Zhangsan" | 3 | 25 | "Lisi" | + | "Zhangsan" | 4 | 35 | "Lisi" | + # delete all edges with different ranks + When executing query: + """ + GO FROM "Zhangsan" OVER friend YIELD id($^) AS src, friend._rank AS rank, friend._dst AS dst + | DELETE EDGE friend $-.src -> $-.dst @ $-.rank; + """ + Then the execution should be successful + # check result + When executing query: + """ + GO FROM "Zhangsan" OVER friend + YIELD $^.person.name, friend.intimacy, friend._rank, friend._dst + """ + Then the result should be, in any order: + | $^.person.name | friend.intimacy | friend._rank | friend._dst | + Then drop the used space + Scenario: delete edges use pipe Given load "nba" csv data to a new space # test delete with pipe wrong vid type diff --git a/tests/tck/features/delete/DeleteTag.IntVid.feature b/tests/tck/features/delete/DeleteTag.IntVid.feature index 1bf23a2d956..11b106c7db9 100644 --- a/tests/tck/features/delete/DeleteTag.IntVid.feature +++ b/tests/tck/features/delete/DeleteTag.IntVid.feature @@ -200,6 +200,7 @@ Feature: Delete int vid of tag """ Then the execution should be successful # after delete tag + # the output has one row because the vertex has multiple tags When executing query: """ FETCH PROP ON player hash("Tim Duncan") YIELD player.name, player.age diff --git a/tests/tck/features/delete/DeleteTag.feature b/tests/tck/features/delete/DeleteTag.feature index 7a4fe136e94..3afecefa036 100644 --- a/tests/tck/features/delete/DeleteTag.feature +++ b/tests/tck/features/delete/DeleteTag.feature @@ -134,7 +134,7 @@ Feature: Delete string vid of tag Then the result should be, in any order: | id | | "Tim Duncan" | - # delete one tag + # delete all tag When executing query: """ DELETE TAG * FROM "Tim Duncan"; @@ -200,6 +200,7 @@ Feature: Delete string vid of tag """ Then the execution should be successful # after delete tag + # the output has one row because the vertex has multiple tags When executing query: """ FETCH PROP ON player "Tim Duncan" YIELD player.name, player.age diff --git a/tests/tck/features/delete/DeleteVertex.feature b/tests/tck/features/delete/DeleteVertex.feature index 58703d63a86..8cc9864d39d 100644 --- a/tests/tck/features/delete/DeleteVertex.feature +++ b/tests/tck/features/delete/DeleteVertex.feature @@ -108,7 +108,7 @@ Feature: Delete string vid of vertex | like._dst | | "Kobe Bryant" | | "Grant Hill" | - | -"Rudy Gay" | + | "Rudy Gay" | # before delete hash id vertex to check value by go When executing query: """ diff --git a/tests/tck/features/insert/Insert.IntVid.feature b/tests/tck/features/insert/Insert.IntVid.feature index 34635dd3798..2ba6e2075db 100644 --- a/tests/tck/features/insert/Insert.IntVid.feature +++ b/tests/tck/features/insert/Insert.IntVid.feature @@ -236,6 +236,34 @@ Feature: Insert int vid of vertex and edge VALUES hash("Laura"):("Laura", 8, "three", 20190901008),hash("Amber"):("Amber", 9, "four", 20180901003) """ Then the execution should be successful + When executing query: + """ + FETCH PROP ON person hash("Laura") YIELD person.name, person.age + """ + Then the result should be, in any order: + | person.name | person.age | + | 'Laura' | 8 | + When executing query: + """ + FETCH PROP ON student hash("Laura") YIELD student.grade, student.number + """ + Then the result should be, in any order: + | student.grade | student.number | + | 'three' | 20190901008 | + When executing query: + """ + FETCH PROP ON person hash("Amber") YIELD person.name, person.age + """ + Then the result should be, in any order: + | person.name | person.age | + | 'Amber' | 9 | + When executing query: + """ + FETCH PROP ON student hash("Amber") YIELD student.grade, student.number + """ + Then the result should be, in any order: + | student.grade | student.number | + | 'four' | 20180901003 | # insert multi vertex one tag When executing query: """ @@ -586,3 +614,15 @@ Feature: Insert int vid of vertex and edge | src | dst | | 300 | 400 | Then drop the used space + + Scenario: insert vertex with non existent tag + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(10) | + When try to execute query: + """ + INSERT VERTEX invalid_vertex VALUES "non_existed_tag":() + """ + Then a SemanticError should be raised at runtime: No schema found diff --git a/tests/tck/features/insert/Insert.feature b/tests/tck/features/insert/Insert.feature index 746922951d5..2a3785cec9f 100644 --- a/tests/tck/features/insert/Insert.feature +++ b/tests/tck/features/insert/Insert.feature @@ -95,7 +95,7 @@ Feature: Insert string vid of vertex and edge INSERT EDGE schoolmate(likeness, nickname) VALUES "Tom"->"Lucy":(85, "Lily") """ Then the execution should be successful - # insert vertex wrong type + # insert edge wrong type When executing query: """ INSERT EDGE schoolmate(likeness, nickname) VALUES "Laura"->"Amber":("87", ""); @@ -224,6 +224,34 @@ Feature: Insert string vid of vertex and edge "Amber":("Amber", 9, "four", 20180901003); """ Then the execution should be successful + When executing query: + """ + FETCH PROP ON person "Laura" YIELD person.name, person.age + """ + Then the result should be, in any order: + | person.name | person.age | + | 'Laura' | 8 | + When executing query: + """ + FETCH PROP ON student "Laura" YIELD student.grade, student.number + """ + Then the result should be, in any order: + | student.grade | student.number | + | 'three' | 20190901008 | + When executing query: + """ + FETCH PROP ON person "Amber" YIELD person.name, person.age + """ + Then the result should be, in any order: + | person.name | person.age | + | 'Amber' | 9 | + When executing query: + """ + FETCH PROP ON student "Amber" YIELD student.grade, student.number + """ + Then the result should be, in any order: + | student.grade | student.number | + | 'four' | 20180901003 | # insert multi vertex one tag When executing query: """ @@ -506,6 +534,22 @@ Feature: Insert string vid of vertex and edge Then the result should be, in any order, with relax comparison: | node | | ('English') | + # insert Chinese character + # if the Chinese character is out of the fixed_string's size, it will be truncated, + # and no invalid char should be inserted + When executing query: + """ + INSERT VERTEX course(name) VALUES "Chinese":("中文字符") + """ + Then the execution should be successful + # check result + When executing query: + """ + FETCH PROP ON course "Chinese" YIELD vertex as node + """ + Then the result should be, in any order, with relax comparison: + | node | + | ('Chinese') | # check result When executing query: """ @@ -523,6 +567,7 @@ Feature: Insert string vid of vertex and edge | course.name | course.introduce | | 'Engli' | NULL | | 'Math' | NULL | + | '中' | NULL | When executing query: """ LOOKUP ON student YIELD student.name, student.age @@ -630,3 +675,15 @@ Feature: Insert string vid of vertex and edge """ Then a ExecutionError should be raised at runtime: Storage Error: The VID must be a 64-bit integer or a string fitting space vertex id length limit. Then drop the used space + + Scenario: insert vertex with non existent tag + Given an empty graph + And create a space with following options: + | partition_num | 1 | + | replica_factor | 1 | + | vid_type | FIXED_STRING(10) | + When try to execute query: + """ + INSERT VERTEX invalid_vertex VALUES "non_existed_tag":() + """ + Then a SemanticError should be raised at runtime: No schema found diff --git a/tests/tck/features/insert/InsertEdgeOnDiffParts.feature b/tests/tck/features/insert/InsertEdgeOnDiffParts.feature index 9c33e3f058e..d1079a47700 100644 --- a/tests/tck/features/insert/InsertEdgeOnDiffParts.feature +++ b/tests/tck/features/insert/InsertEdgeOnDiffParts.feature @@ -1,7 +1,7 @@ # Copyright (c) 2021 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License. -Feature: Insert vertex and edge with if not exists +Feature: Insert edge on different parts Scenario: insert edge with default value Given an empty graph From 44e1dea82cf3cdcdf2e81f6ba7985593f2d6ad67 Mon Sep 17 00:00:00 2001 From: George <58841610+Shinji-IkariG@users.noreply.github.com> Date: Sat, 28 Jan 2023 14:34:27 +0800 Subject: [PATCH 48/51] fix third party version in package.sh (#5281) The dump_syms tool path should be match with third party version. --- package/package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/package.sh b/package/package.sh index 086eeed03ec..7a006ee6221 100755 --- a/package/package.sh +++ b/package/package.sh @@ -197,8 +197,8 @@ function package { function _find_dump_syms_tool { if [[ -x ${build_dir}/third-party/install/bin/dump_syms ]]; then dump_syms_tool_dir=${build_dir}/third-party/install/bin - elif [[ -x /opt/vesoft/third-party/2.0/bin/dump_syms ]]; then - dump_syms_tool_dir=/opt/vesoft/third-party/2.0/bin + elif [[ -x /opt/vesoft/third-party/3.3/bin/dump_syms ]]; then + dump_syms_tool_dir=/opt/vesoft/third-party/3.3/bin else echo ">>> Failed to find the dump_syms tool <<<" exit 1 From 8c2666dbfb390925af1867ce7ecf8538f1c8cd4d Mon Sep 17 00:00:00 2001 From: shylock <33566796+Shylock-Hg@users.noreply.github.com> Date: Sun, 29 Jan 2023 14:19:51 +0800 Subject: [PATCH 49/51] Test/user (#5139) * Add some tests about user management. * Add tests about user roles. * Format. * Fix tck fixture name. * Fix step name. * Change step name. --------- Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> --- tests/tck/conftest.py | 16 ++ tests/tck/features/user/User.feature | 410 ++++++++++++++++++++++++++- 2 files changed, 424 insertions(+), 2 deletions(-) diff --git a/tests/tck/conftest.py b/tests/tck/conftest.py index 03f159ad36f..322f687414c 100644 --- a/tests/tck/conftest.py +++ b/tests/tck/conftest.py @@ -980,3 +980,19 @@ def switch_to_new_session(conn_pool, user, password, class_fixture_variables, ex sess = conn_pool.get_session(user, password) class_fixture_variables["sessions"].append(sess) exec_ctx["current_session"] = sess + +@when(parse('verify login with user "{user}"')) +def login_without_password(conn_pool, user): + sess = None + try: + sess = conn_pool.get_session(user, '') + except Exception as e: + assert e + +@when(parse('verify login with user "{user}" and password "{password}"')) +def login_with_password(conn_pool, user, password): + sess = None + try: + sess = conn_pool.get_session(user, password) + except Exception as e: + assert e diff --git a/tests/tck/features/user/User.feature b/tests/tck/features/user/User.feature index 4703fb05a6c..9c1a6d1f4d5 100644 --- a/tests/tck/features/user/User.feature +++ b/tests/tck/features/user/User.feature @@ -9,11 +9,24 @@ Feature: User & privilege Test DROP USER IF EXISTS user1 """ Then the execution should be successful + When executing query: + """ + DROP USER user1 + """ + Then a ExecutionError should be raised at runtime: User not existed! When executing query: """ CREATE USER user1 WITH PASSWORD "pwd1" """ Then the execution should be successful + When executing query: + """ + DESC USER user1; + """ + Then the result should be, in any order: + | role | space | + And wait 6 seconds + When verify login with user "user2" and password "pwd1" When executing query: """ DROP USER IF EXISTS user2 @@ -21,9 +34,21 @@ Feature: User & privilege Test Then the execution should be successful When executing query: """ - CREATE USER user2 + CREATE USER IF NOT EXISTS user2 + """ + Then the execution should be successful + When executing query: + """ + CREATE USER IF NOT EXISTS user2 """ Then the execution should be successful + When executing query: + """ + CREATE USER user2 + """ + Then a ExecutionError should be raised at runtime: + And wait 6 seconds + When verify login with user "user2" When executing query: """ CREATE USER user1 WITH PASSWORD "pwd1" @@ -43,6 +68,311 @@ Feature: User & privilege Test | "root" | | "user1" | | "user2" | + When executing query: + """ + CREATE USER u + """ + Then the execution should be successful + When executing query: + """ + CREATE USER u123456789ABCDEF + """ + Then the execution should be successful + When executing query: + """ + CREATE USER u123456789ABCDEFG + """ + Then a SemanticError should be raised at runtime: Username exceed maximum length 16 characters. + When executing query: + """ + CREATE USER 123456 + """ + Then a SyntaxError should be raised at runtime: syntax error near `123456' + When executing query: + """ + CREATE USER u&b + """ + Then a SyntaxError should be raised at runtime: syntax error near `&b' + When executing query: + """ + CREATE USER `用户A` + """ + Then the execution should be successful + And wait 6 seconds + When verify login with user "用户A" + When executing query: + """ + CREATE USER A1; + CREATE USER a1; + """ + Then the execution should be successful + And wait 6 seconds + When verify login with user "A1" + When verify login with user "a1" + When executing query: + """ + CREATE USER `CREATE`; + CREATE USER `ROLE`; + """ + Then the execution should be successful + When verify login with user "CREATE" + When verify login with user "ROLE" + When executing query: + """ + CREATE USER u3 WITH PASSWORD "012345678910111213141516"; + CREATE USER u4 WITH PASSWORD "0"; + """ + Then the execution should be successful + And wait 6 seconds + When verify login with user "u3" and password "012345678910111213141516" + When verify login with user "u4" and password "0" + When executing query: + """ + CREATE USER u5 WITH PASSWORD "0123456789101112131415161"; + """ + Then a SemanticError should be raised at runtime: Password exceed maximum length 24 characters. + When executing query: + """ + CREATE USER u6 WITH PASSWORD "中文密码^*()12"; + """ + Then the execution should be successful + And wait 6 seconds + When verify login with user "u6" and password "中文密码^*()12" + When executing query: + """ + DROP USER IF EXISTS u6; + """ + Then the execution should be successful + # TODO(shylock) fix it + # When executing query: + # """ + # DESC USER u6; + # """ + # Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + DROP USER IF EXISTS u6; + """ + Then the execution should be successful + When executing query: + """ + DROP USER u6; + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + DROP USER root; + """ + Then a SemanticError should be raised at runtime: Can't drop root user. + + Scenario: User roles + When executing query: + """ + CREATE USER user_mlt_roles; + GRANT ROLE USER ON nba TO user_mlt_roles; + GRANT ROLE GUEST ON student TO user_mlt_roles; + """ + Then the execution should be successful + When executing query: + """ + DESC USER user_mlt_roles; + """ + Then the result should be, in any order: + | role | space | + | "USER" | "nba" | + | "GUEST" | "student" | + When executing query: + """ + DROP USER user_mlt_roles; + """ + Then the execution should be successful + # TODO(shylock) fix me + # When executing query: + # """ + # DESC USER user_mlt_roles + # """ + # Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + CREATE USER user_mlt_roles; + """ + Then the execution should be successful + When executing query: + """ + GRANT ROLE ADMIN ON nba TO user_mlt_roles; + GRANT ROLE ADMIN ON student TO user_mlt_roles; + GRANT ROLE GUEST ON nba_int_vid TO user_mlt_roles; + """ + Then the execution should be successful + When executing query: + """ + GRANT ROLE DBA ON nba TO user_mlt_roles; + """ + Then the execution should be successful + When executing query: + """ + DESC USER user_mlt_roles; + """ + Then the result should be, in any order: + | role | space | + | "DBA" | "nba" | + | "ADMIN" | "student" | + | "GUEST" | "nba_int_vid" | + When executing query: + """ + GRANT ROLE ADMIN ON nba TO user_mlt_roles; + GRANT ROLE GUEST ON nba TO user_mlt_roles; + GRANT ROLE USER ON nba TO user_mlt_roles; + """ + Then the execution should be successful + When executing query: + """ + SHOW ROLES IN nba + """ + Then the result should be, in any order: + | Account | Role Type | + | "test_permission" | "USER" | + | "user_mlt_roles" | "USER" | + When executing query: + """ + DESC USER user_mlt_roles; + """ + Then the result should be, in any order: + | role | space | + | "USER" | "nba" | + | "ADMIN" | "student" | + | "GUEST" | "nba_int_vid" | + When executing query: + """ + GRANT ROLE ADMIN ON not_exists TO user_mlt_roles; + """ + Then a ExecutionError should be raised at runtime: SpaceNotFound: SpaceName `not_exists` + When executing query: + """ + GRANT ROLE ADMIN ON nba TO not_exists; + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + GRANT not_exists ADMIN ON nba TO user_mlt_roles; + """ + Then a SyntaxError should be raised at runtime: syntax error near `not_exists' + When executing query: + """ + GRANT GOD ON nba TO user_mlt_roles; + """ + Then a PermissionError should be raised at runtime: No permission to grant/revoke god user. + When executing query: + """ + REVOKE ROLE USER ON nba FROM user_mlt_roles; + """ + Then the execution should be successful + When executing query: + """ + DESC USER user_mlt_roles; + """ + Then the result should be, in any order: + | role | space | + | "ADMIN" | "student" | + | "GUEST" | "nba_int_vid" | + When executing query: + """ + REVOKE ROLE GUEST ON student FROM user_mlt_roles; + """ + Then a ExecutionError should be raised at runtime: Improper role! + When executing query: + """ + REVOKE ROLE ADMIN ON nba FROM user_mlt_roles; + """ + Then a ExecutionError should be raised at runtime: Role not existed! + When executing query: + """ + REVOKE ROLE not_exists ON nba FROM user_mlt_roles; + """ + Then a SyntaxError should be raised at runtime: syntax error near `not_exists' + When executing query: + """ + REVOKE ROLE GOD ON nba FROM root; + """ + Then a PermissionError should be raised at runtime: Permission denied + When executing query: + """ + REVOKE ROLE USER ON not_exists FROM user_mlt_roles; + """ + Then a ExecutionError should be raised at runtime: SpaceNotFound: SpaceName `not_exists` + When executing query: + """ + REVOKE ROLE USER ON nba FROM not_exists; + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + DROP USER user_mlt_roles + """ + Then the execution should be successful + When executing query: + """ + SHOW ROLES IN nba + """ + Then the result should be, in any order: + | Account | Role Type | + | "test_permission" | "USER" | + When executing query: + """ + GRANT ROLE ADMIN ON nba TO user_mlt_roles; + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + REVOKE ROLE ADMIN ON nba FROM user_mlt_roles; + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + GRANT GUEST ON nba TO root; + """ + Then a SemanticError should be raised at runtime: User 'root' is GOD, cannot be granted. + When executing query: + """ + SHOW ROLES IN not_exists + """ + Then a ExecutionError should be raised at runtime: SpaceNotFound: SpaceName `not_exists` + + Scenario: Recreate space roles + When executing query: + """ + CREATE SPACE test_roles(partition_num=1, replica_factor=1, vid_type=int64); + CREATE USER IF NOT EXISTS user_roles WITH PASSWORD "pwd"; + """ + Then the execution should be successful + And wait 6 seconds + When executing query: + """ + GRANT ROLE ADMIN ON test_roles TO user_roles; + """ + Then the execution should be successful + When executing query: + """ + DROP SPACE test_roles; + """ + Then the execution should be successful + When executing query: + """ + CREATE SPACE test_roles(partition_num=1, replica_factor=1, vid_type=int64); + """ + Then the execution should be successful + And wait 6 seconds + When executing query: + """ + SHOW ROLES IN test_roles + """ + Then the result should be, in any order: + | Account | Role Type | + When executing query: + """ + DROP SPACE test_roles + """ + Then the execution should be successful Scenario: Alter user When executing query: @@ -65,16 +395,75 @@ Feature: User & privilege Test ALTER USER user2 WITH PASSWORD "pwd1" """ Then the execution should be successful + And wait 6 seconds + When verify login with user "user2" and password "pwd1" When executing query: """ CHANGE PASSWORD user2 FROM "pwd2" TO "pwd1" """ Then a ExecutionError should be raised at runtime: Invalid password! + When executing query: + """ + CHANGE PASSWORD user2 FROM "pwd1" TO "01234567890111213141516171" + """ + Then a SemanticError should be raised at runtime: New password exceed maximum length 24 characters. When executing query: """ CHANGE PASSWORD user2 FROM "pwd1" TO "pwd2" """ Then the execution should be successful + When executing query: + """ + CREATE USER IF NOT EXISTS u7 + """ + Then the execution should be successful + When executing query: + """ + ALTER USER u7 WITH PASSWORD "pwd1" + """ + Then the execution should be successful + And wait 6 seconds + When verify login with user "u7" and password "pwd1" + When executing query: + """ + ALTER USER u7 WITH PASSWORD "0123456789011121314151617" + """ + Then a SemanticError should be raised at runtime: Password exceed maximum length 24 characters. + When executing query: + """ + DROP USER IF EXISTS u7 + """ + Then the execution should be successful + When executing query: + """ + CHANGE PASSWORD u7 FROM "pwd1" TO "nebula" + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + ALTER USER not_exists WITH PASSWORD "pwd1" + """ + Then a ExecutionError should be raised at runtime: User not existed! + When executing query: + """ + CHANGE PASSWORD root FROM "nebula" TO "root" + """ + Then the execution should be successful + When executing query: + """ + CHANGE PASSWORD root FROM "root" TO "nebula" + """ + Then the execution should be successful + When executing query: + """ + CREATE USER IF NOT EXISTS u8 + """ + Then the execution should be successful + When executing query: + """ + CHANGE PASSWORD u8 FROM "" TO "pwd2" + """ + Then the execution should be successful Scenario: Drop user When executing query: @@ -281,7 +670,6 @@ Feature: User & privilege Test """ Then the execution should be successful - @skip Scenario: Describe User When executing query: """ @@ -352,6 +740,17 @@ Feature: User & privilege Test Then the result should be, in any order, with relax comparison: | role | space | | "ADMIN" | "user_tmp_space_4" | + When executing query: + """ + REVOKE ROLE ADMIN ON user_tmp_space_4 FROM user1 + """ + Then the execution should be successful + When executing query: + """ + DESC USER user1 + """ + Then the result should be, in any order: + | role | space | When executing query with user user1 with password pwd1: """ DESC USER user2 @@ -367,3 +766,10 @@ Feature: User & privilege Test DESC USER root """ Then a PermissionError should be raised at runtime: + +# TODO(shylock) fix it +# When executing query: +# """ +# DESCRIBE USER not_exists +# """ +# Then a ExecutionError should be raised at runtime: User not existed! From 91d7c657f5b788be003524a47a91de78819fa2fc Mon Sep 17 00:00:00 2001 From: "hs.zhang" <22708345+cangfengzhs@users.noreply.github.com> Date: Sun, 29 Jan 2023 15:14:06 +0800 Subject: [PATCH 50/51] fix https (#5283) --- src/common/http/HttpClient.cpp | 8 ++++++++ src/common/http/HttpClient.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/common/http/HttpClient.cpp b/src/common/http/HttpClient.cpp index 3ef01d93619..c3ab96ad0bd 100644 --- a/src/common/http/HttpClient.cpp +++ b/src/common/http/HttpClient.cpp @@ -91,6 +91,9 @@ HttpResponse HttpClient::sendRequest(const std::string& url, CURL* curl = curl_easy_init(); CHECK(curl); setUrl(curl, url); + if (folly::StringPiece(url).startsWith("https")) { + setSSL(curl); + } setMethod(curl, method); curl_slist* h = setHeaders(curl, header); if (!body.empty()) { @@ -156,6 +159,11 @@ void HttpClient::setAuth(CURL* curl, const std::string& username, const std::str } } +void HttpClient::setSSL(CURL* curl) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); +} + size_t HttpClient::onWriteData(void* ptr, size_t size, size_t nmemb, void* stream) { if (ptr == nullptr || size == 0) { return 0; diff --git a/src/common/http/HttpClient.h b/src/common/http/HttpClient.h index 9c041fcc012..5eebeb52b27 100644 --- a/src/common/http/HttpClient.h +++ b/src/common/http/HttpClient.h @@ -89,6 +89,7 @@ class HttpClient { static void setRespBody(CURL* curl, const std::string& body); static void setRespHeader(CURL* curl, const std::string& header); static void setTimeout(CURL* curl); + static void setSSL(CURL* curl); static void setAuth(CURL* curl, const std::string& username, const std::string& password); static size_t onWriteData(void* ptr, size_t size, size_t nmemb, void* stream); }; From 022e6e7f49b396999de1326b6c73a852f163f950 Mon Sep 17 00:00:00 2001 From: codesigner Date: Sun, 29 Jan 2023 16:15:04 +0800 Subject: [PATCH 51/51] fix memtracker bugs during stress test on graphd and storaged (#5276) * fix memtracker bugs during stress test on graphd and storaged * fix lint * fix RocksEngine memory leak of raw pointer iter * add ENABLE_MEMORY_TRACKER build option & support adaptive limit for MemoryTracker * delete debug log * refine log * refine log * fix build * refine error log * print warning if memtracker is off * fix rocksdb leak by turn off memcheck * refine synamic-self-adaptive * fix cmake check * minor * minor * minor * minor * minor * refine double equel compare --- .github/workflows/pull_request.yml | 1 + CMakeLists.txt | 15 ++ cmake/nebula/GeneralCMakeOptions.cmake | 1 + src/clients/meta/MetaClient.cpp | 162 ++++++++++++++++++ src/clients/storage/StorageClientBase-inl.h | 1 + src/codec/RowReader.cpp | 4 +- src/codec/RowReader.h | 12 +- src/codec/RowReaderV1.cpp | 36 ++-- src/codec/RowReaderV1.h | 36 ++-- src/codec/RowReaderV2.cpp | 6 +- src/codec/RowReaderV2.h | 6 +- src/codec/RowReaderWrapper.cpp | 7 +- src/codec/RowReaderWrapper.h | 16 +- src/codec/RowWriterV2.cpp | 60 +++---- src/codec/RowWriterV2.h | 78 ++++----- src/common/base/ObjectPool.h | 9 +- src/common/datatypes/DataSet.h | 4 +- src/common/datatypes/Edge.cpp | 2 +- src/common/datatypes/Edge.h | 2 +- src/common/datatypes/Geography.cpp | 2 +- src/common/datatypes/Geography.h | 2 +- src/common/datatypes/List.h | 2 +- src/common/datatypes/Map.cpp | 2 +- src/common/datatypes/Map.h | 2 +- src/common/datatypes/Path.cpp | 2 +- src/common/datatypes/Path.h | 12 +- src/common/datatypes/Set.cpp | 2 +- src/common/datatypes/Set.h | 2 +- src/common/datatypes/Value.cpp | 104 +++++------ src/common/datatypes/Value.h | 6 +- src/common/datatypes/ValueOps-inl.h | 12 +- src/common/datatypes/Vertex.cpp | 2 +- src/common/datatypes/Vertex.h | 2 +- src/common/expression/Expression.cpp | 20 +-- src/common/expression/Expression.h | 20 +-- src/common/geo/GeoIndex.cpp | 16 +- src/common/geo/GeoIndex.h | 16 +- src/common/memory/Memory.h | 10 ++ src/common/memory/MemoryTracker.cpp | 4 +- src/common/memory/MemoryTracker.h | 36 +++- src/common/memory/MemoryUtils.cpp | 141 ++++++++++----- src/common/memory/MemoryUtils.h | 6 + src/common/memory/NewDelete.cpp | 18 +- src/graph/context/ExecutionContext.cpp | 12 +- src/graph/context/ExecutionContext.h | 3 + src/graph/context/Symbols.cpp | 18 +- src/graph/context/Symbols.h | 2 + .../executor/admin/SwitchSpaceExecutor.cpp | 8 + .../executor/algo/BFSShortestPathExecutor.cpp | 1 - src/graph/gc/GC.cpp | 7 +- src/graph/optimizer/rule/IndexScanRule.cpp | 2 +- src/graph/optimizer/rule/IndexScanRule.h | 2 +- .../AsyncMsgNotifyBasedScheduler.cpp | 107 +++++++++++- .../scheduler/AsyncMsgNotifyBasedScheduler.h | 25 +++ src/graph/scheduler/Scheduler.h | 2 + src/graph/service/QueryEngine.cpp | 1 - src/graph/service/QueryInstance.cpp | 3 +- src/graph/service/RequestContext.h | 2 +- src/kvstore/RocksEngine.cpp | 28 +-- src/kvstore/RocksEngine.h | 10 ++ .../listener/elasticsearch/ESListener.cpp | 4 +- .../listener/elasticsearch/ESListener.h | 2 +- src/kvstore/raftex/RaftPart.cpp | 8 +- src/storage/BaseProcessor.h | 3 +- src/storage/GraphStorageServiceHandler.cpp | 28 +-- src/storage/index/LookupProcessor.cpp | 76 ++++---- src/storage/query/GetDstBySrcProcessor.cpp | 88 +++++----- src/storage/query/GetNeighborsProcessor.cpp | 71 ++++---- src/storage/query/GetPropProcessor.cpp | 72 ++++---- src/storage/query/ScanEdgeProcessor.cpp | 74 ++++---- src/storage/query/ScanVertexProcessor.cpp | 73 ++++---- 71 files changed, 1046 insertions(+), 585 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 85df78b2182..ff1d72c25f0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -132,6 +132,7 @@ jobs: -DCMAKE_CXX_COMPILER=$TOOLSET_CLANG_DIR/bin/clang++ \ -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/clang \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_MEMORY_TRACKER=off \ -DENABLE_ASAN=on \ -DENABLE_TESTING=on \ -GNinja \ diff --git a/CMakeLists.txt b/CMakeLists.txt index d5c55196392..6fdb7b827d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,21 @@ if (ENABLE_NATIVE) add_compile_options(-fPIC) endif() +if(ENABLE_MEMORY_TRACKER) + if(ENABLE_JEMALLOC) + if(NOT ENABLE_ASAN) + add_definitions(-DENABLE_MEMORY_TRACKER) + message(STATUS "MemoryTracker is ENABLED") + else() + message(FATAL_ERROR "MemoryTracker need -DENABLE_ASAN=off") + endif() + else() + message(FATAL_ERROR "MemoryTracker need -DENABLE_JEMALLOC=on") + endif() +else() + message(WARNING "MemoryTracker is DISABLED") +endif() + include_directories(AFTER ${CMAKE_SOURCE_DIR}/src) include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/src) diff --git a/cmake/nebula/GeneralCMakeOptions.cmake b/cmake/nebula/GeneralCMakeOptions.cmake index aaf3f1ea0e6..0e3c855b3e9 100644 --- a/cmake/nebula/GeneralCMakeOptions.cmake +++ b/cmake/nebula/GeneralCMakeOptions.cmake @@ -20,6 +20,7 @@ option(ENABLE_GDB_SCRIPT_SECTION "Add .debug_gdb_scripts section" OFF) option(DISABLE_CXX11_ABI "Whether to disable cxx11 abi" OFF) option(ENABLE_BREAKPAD "Whether to enable breakpad" OFF) option(ENABLE_STANDALONE_VERSION "Enable standalone version build" OFF) +option(ENABLE_MEMORY_TRACKER "Enable memory tracker" ON) get_cmake_property(variable_list VARIABLES) foreach(_varname ${variable_list}) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index e04cb91214e..8d0125099e1 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -254,6 +254,7 @@ bool MetaClient::loadUsersAndRoles() { } bool MetaClient::loadData() { + memory::MemoryCheckOffGuard g; // UNKNOWN role will skip heartbeat if (options_.role_ != cpp2::HostRole::UNKNOWN && localDataLastUpdateTime_ == metadLastUpdateTime_) { @@ -425,6 +426,7 @@ bool MetaClient::loadData() { } TagSchemas MetaClient::buildTagSchemas(std::vector tagItemVec) { + memory::MemoryCheckOffGuard g; TagSchemas tagSchemas; for (auto& tagIt : tagItemVec) { // meta will return the different version from new to old @@ -447,6 +449,7 @@ TagSchemas MetaClient::buildTagSchemas(std::vector tagItemVec) { } EdgeSchemas MetaClient::buildEdgeSchemas(std::vector edgeItemVec) { + memory::MemoryCheckOffGuard g; EdgeSchemas edgeSchemas; std::unordered_set> edges; for (auto& edgeIt : edgeItemVec) { @@ -470,6 +473,7 @@ EdgeSchemas MetaClient::buildEdgeSchemas(std::vector edgeItemVec } void MetaClient::addSchemaField(NebulaSchemaProvider* schema, const cpp2::ColumnDef& col) { + memory::MemoryCheckOffGuard g; bool hasDef = col.default_value_ref().has_value(); auto& colType = col.get_type(); size_t len = colType.type_length_ref().has_value() ? *colType.get_type_length() : 0; @@ -493,6 +497,7 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, SpaceNewestTagVerMap& newestTagVerMap, SpaceNewestEdgeVerMap& newestEdgeVerMap, SpaceAllEdgeMap& allEdgeMap) { + memory::MemoryCheckOffGuard g; auto tagRet = listTagSchemas(spaceId).get(); if (!tagRet.ok()) { LOG(ERROR) << "Get tag schemas failed for spaceId " << spaceId << ", " << tagRet.status(); @@ -559,6 +564,7 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, } Indexes buildIndexes(std::vector indexItemVec) { + memory::MemoryCheckOffGuard g; Indexes indexes; for (auto index : indexItemVec) { auto indexName = index.get_index_name(); @@ -570,6 +576,7 @@ Indexes buildIndexes(std::vector indexItemVec) { } bool MetaClient::loadIndexes(GraphSpaceID spaceId, std::shared_ptr cache) { + memory::MemoryCheckOffGuard g; auto tagIndexesRet = listTagIndexes(spaceId).get(); if (!tagIndexesRet.ok()) { LOG(ERROR) << "Get tag indexes failed for spaceId " << spaceId << ", " @@ -607,6 +614,7 @@ bool MetaClient::loadIndexes(GraphSpaceID spaceId, std::shared_ptr cache) { + memory::MemoryCheckOffGuard g; auto listenerRet = listListener(spaceId).get(); if (!listenerRet.ok()) { LOG(ERROR) << "Get listeners failed for spaceId " << spaceId << ", " << listenerRet.status(); @@ -622,6 +630,7 @@ bool MetaClient::loadListeners(GraphSpaceID spaceId, std::shared_ptr> MetaClient::reverse( const PartsAlloc& parts) { + memory::MemoryCheckOffGuard g; std::unordered_map> hosts; for (auto& partHost : parts) { for (auto& h : partHost.second) { @@ -694,6 +707,7 @@ void MetaClient::getResponse(Request req, bool toLeader, int32_t retry, int32_t retryLimit) { + memory::MemoryCheckOffGuard g; stats::StatsManager::addValue(kNumRpcSentToMetad); auto* evb = ioThreadPool_->getEventBase(); HostAddr host; @@ -803,6 +817,7 @@ void MetaClient::getResponse(Request req, } std::vector MetaClient::toSpaceIdName(const std::vector& tIdNames) { + memory::MemoryCheckOffGuard g; std::vector idNames; idNames.resize(tIdNames.size()); std::transform(tIdNames.begin(), tIdNames.end(), idNames.begin(), [](const auto& tin) { @@ -813,6 +828,7 @@ std::vector MetaClient::toSpaceIdName(const std::vector Status MetaClient::handleResponse(const RESP& resp) { + memory::MemoryCheckOffGuard g; switch (resp.get_code()) { case nebula::cpp2::ErrorCode::SUCCEEDED: return Status::OK(); @@ -979,6 +995,7 @@ Status MetaClient::handleResponse(const RESP& resp) { } PartsMap MetaClient::doGetPartsMap(const HostAddr& host, const LocalCache& localCache) { + memory::MemoryCheckOffGuard g; PartsMap partMap; for (const auto& it : localCache) { auto spaceId = it.first; @@ -999,6 +1016,7 @@ PartsMap MetaClient::doGetPartsMap(const HostAddr& host, const LocalCache& local } void MetaClient::diff(const LocalCache& oldCache, const LocalCache& newCache) { + memory::MemoryCheckOffGuard g; folly::SharedMutex::WriteHolder holder(listenerLock_); if (listener_ == nullptr) { VLOG(3) << "Listener is null!"; @@ -1060,6 +1078,7 @@ void MetaClient::diff(const LocalCache& oldCache, const LocalCache& newCache) { } void MetaClient::listenerDiff(const LocalCache& oldCache, const LocalCache& newCache) { + memory::MemoryCheckOffGuard g; folly::SharedMutex::WriteHolder holder(listenerLock_); if (listener_ == nullptr) { VLOG(3) << "Listener is null!"; @@ -1169,6 +1188,7 @@ void MetaClient::listenerDiff(const LocalCache& oldCache, const LocalCache& newC } void MetaClient::loadRemoteListeners() { + memory::MemoryCheckOffGuard g; folly::SharedMutex::WriteHolder holder(listenerLock_); if (listener_ == nullptr) { VLOG(3) << "Listener is null!"; @@ -1194,6 +1214,7 @@ void MetaClient::loadRemoteListeners() { /// ================================== public methods ================================= PartitionID MetaClient::partId(int32_t numParts, const VertexID id) const { + memory::MemoryCheckOffGuard g; // If the length of the id is 8, we will treat it as int64_t to be compatible // with the version 1.0 uint64_t vid = 0; @@ -1210,6 +1231,7 @@ PartitionID MetaClient::partId(int32_t numParts, const VertexID id) const { folly::Future> MetaClient::submitJob( GraphSpaceID spaceId, cpp2::JobOp op, cpp2::JobType type, std::vector paras) { + memory::MemoryCheckOffGuard g; cpp2::AdminJobReq req; req.space_id_ref() = spaceId; req.op_ref() = op; @@ -1227,6 +1249,7 @@ folly::Future> MetaClient::submitJob( folly::Future> MetaClient::createSpace(meta::cpp2::SpaceDesc spaceDesc, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateSpaceReq req; req.properties_ref() = std::move(spaceDesc); req.if_not_exists_ref() = ifNotExists; @@ -1242,6 +1265,7 @@ folly::Future> MetaClient::createSpace(meta::cpp2::SpaceD folly::Future> MetaClient::createSpaceAs(const std::string& oldSpaceName, const std::string& newSpaceName) { + memory::MemoryCheckOffGuard g; cpp2::CreateSpaceAsReq req; req.old_space_name_ref() = oldSpaceName; req.new_space_name_ref() = newSpaceName; @@ -1256,6 +1280,7 @@ folly::Future> MetaClient::createSpaceAs(const std::strin } folly::Future>> MetaClient::listSpaces() { + memory::MemoryCheckOffGuard g; cpp2::ListSpacesReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -1270,6 +1295,7 @@ folly::Future>> MetaClient::listSpaces() { } folly::Future> MetaClient::getSpace(std::string name) { + memory::MemoryCheckOffGuard g; cpp2::GetSpaceReq req; req.space_name_ref() = std::move(name); folly::Promise> promise; @@ -1283,6 +1309,7 @@ folly::Future> MetaClient::getSpace(std::string name) } folly::Future> MetaClient::dropSpace(std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropSpaceReq req; req.space_name_ref() = std::move(name); req.if_exists_ref() = ifExists; @@ -1299,6 +1326,7 @@ folly::Future> MetaClient::dropSpace(std::string name, const bool } folly::Future> MetaClient::clearSpace(std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::ClearSpaceReq req; req.space_name_ref() = std::move(name); req.if_exists_ref() = ifExists; @@ -1315,6 +1343,7 @@ folly::Future> MetaClient::clearSpace(std::string name, const boo } folly::Future>> MetaClient::listHosts(cpp2::ListHostType tp) { + memory::MemoryCheckOffGuard g; cpp2::ListHostsReq req; req.type_ref() = tp; @@ -1331,6 +1360,7 @@ folly::Future>> MetaClient::listHosts(cpp2: folly::Future> MetaClient::alterSpace(const std::string& spaceName, meta::cpp2::AlterSpaceOp op, const std::vector& paras) { + memory::MemoryCheckOffGuard g; cpp2::AlterSpaceReq req; req.op_ref() = op; req.space_name_ref() = spaceName; @@ -1349,6 +1379,7 @@ folly::Future> MetaClient::alterSpace(const std::string& spaceNam folly::Future>> MetaClient::listParts( GraphSpaceID spaceId, std::vector partIds) { + memory::MemoryCheckOffGuard g; cpp2::ListPartsReq req; req.space_id_ref() = spaceId; req.part_ids_ref() = std::move(partIds); @@ -1364,6 +1395,7 @@ folly::Future>> MetaClient::listParts( folly::Future>>> MetaClient::getPartsAlloc(GraphSpaceID spaceId, PartTerms* partTerms) { + memory::MemoryCheckOffGuard g; cpp2::GetPartsAllocReq req; req.space_id_ref() = spaceId; folly::Promise>>> promise; @@ -1388,6 +1420,7 @@ MetaClient::getPartsAlloc(GraphSpaceID spaceId, PartTerms* partTerms) { } StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1401,6 +1434,7 @@ StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& } StatusOr MetaClient::getSpaceNameByIdFromCache(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1416,6 +1450,7 @@ StatusOr MetaClient::getSpaceNameByIdFromCache(GraphSpaceID spaceId StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1430,6 +1465,7 @@ StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, StatusOr MetaClient::getTagNameByIdFromCache(const GraphSpaceID& space, const TagID& tagId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1444,6 +1480,7 @@ StatusOr MetaClient::getTagNameByIdFromCache(const GraphSpaceID& sp StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1458,6 +1495,7 @@ StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& sp StatusOr MetaClient::getEdgeNameByTypeFromCache(const GraphSpaceID& space, const EdgeType edgeType) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1471,6 +1509,7 @@ StatusOr MetaClient::getEdgeNameByTypeFromCache(const GraphSpaceID& } StatusOr> MetaClient::getAllEdgeFromCache(const GraphSpaceID& space) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1484,12 +1523,14 @@ StatusOr> MetaClient::getAllEdgeFromCache(const GraphSp } PartsMap MetaClient::getPartsMapFromCache(const HostAddr& host) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); return doGetPartsMap(host, metadata.localCache_); } StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1511,6 +1552,7 @@ StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, Part Status MetaClient::checkPartExistInCache(const HostAddr& host, GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1531,6 +1573,7 @@ Status MetaClient::checkPartExistInCache(const HostAddr& host, } Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1546,6 +1589,7 @@ Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spa } StatusOr MetaClient::partsNum(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1559,6 +1603,7 @@ folly::Future> MetaClient::createTagSchema(GraphSpaceID spaceId, std::string name, cpp2::Schema schema, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(name); @@ -1578,6 +1623,7 @@ folly::Future> MetaClient::alterTagSchema(GraphSpaceID spaceId, std::string name, std::vector items, cpp2::SchemaProp schemaProp) { + memory::MemoryCheckOffGuard g; cpp2::AlterTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(name); @@ -1597,6 +1643,7 @@ folly::Future> MetaClient::alterTagSchema(GraphSpaceID spaceId, folly::Future>> MetaClient::listTagSchemas( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListTagsReq req; req.space_id_ref() = spaceId; folly::Promise>> promise; @@ -1612,6 +1659,7 @@ folly::Future>> MetaClient::listTagSchemas( folly::Future> MetaClient::dropTagSchema(GraphSpaceID spaceId, std::string tagName, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(tagName); @@ -1631,6 +1679,7 @@ folly::Future> MetaClient::dropTagSchema(GraphSpaceID spaceId, folly::Future> MetaClient::getTagSchema(GraphSpaceID spaceId, std::string name, int64_t version) { + memory::MemoryCheckOffGuard g; cpp2::GetTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(name); @@ -1649,6 +1698,7 @@ folly::Future> MetaClient::createEdgeSchema(GraphSpaceID spac std::string name, cpp2::Schema schema, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1669,6 +1719,7 @@ folly::Future> MetaClient::alterEdgeSchema(GraphSpaceID spaceId, std::string name, std::vector items, cpp2::SchemaProp schemaProp) { + memory::MemoryCheckOffGuard g; cpp2::AlterEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1688,6 +1739,7 @@ folly::Future> MetaClient::alterEdgeSchema(GraphSpaceID spaceId, folly::Future>> MetaClient::listEdgeSchemas( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListEdgesReq req; req.space_id_ref() = spaceId; folly::Promise>> promise; @@ -1703,6 +1755,7 @@ folly::Future>> MetaClient::listEdgeSchemas folly::Future> MetaClient::getEdgeSchema(GraphSpaceID spaceId, std::string name, SchemaVer version) { + memory::MemoryCheckOffGuard g; cpp2::GetEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1720,6 +1773,7 @@ folly::Future> MetaClient::getEdgeSchema(GraphSpaceID spa folly::Future> MetaClient::dropEdgeSchema(GraphSpaceID spaceId, std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1743,6 +1797,7 @@ folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID bool ifNotExists, const cpp2::IndexParams* indexParams, const std::string* comment) { + memory::MemoryCheckOffGuard g; cpp2::CreateTagIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(indexName); @@ -1769,6 +1824,7 @@ folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID folly::Future> MetaClient::dropTagIndex(GraphSpaceID spaceID, std::string name, bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropTagIndexReq req; req.space_id_ref() = (spaceID); req.index_name_ref() = (std::move(name)); @@ -1788,6 +1844,7 @@ folly::Future> MetaClient::dropTagIndex(GraphSpaceID spaceID, folly::Future> MetaClient::getTagIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::GetTagIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(name); @@ -1804,6 +1861,7 @@ folly::Future> MetaClient::getTagIndex(GraphSpaceID sp folly::Future>> MetaClient::listTagIndexes( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListTagIndexesReq req; req.space_id_ref() = spaceId; @@ -1818,6 +1876,7 @@ folly::Future>> MetaClient::listTagIndexes } folly::Future> MetaClient::rebuildTagIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::RebuildIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(name); @@ -1836,6 +1895,7 @@ folly::Future> MetaClient::rebuildTagIndex(GraphSpaceID spaceID, folly::Future>> MetaClient::listTagIndexStatus( GraphSpaceID spaceID) { + memory::MemoryCheckOffGuard g; cpp2::ListIndexStatusReq req; req.space_id_ref() = spaceID; @@ -1859,6 +1919,7 @@ folly::Future> MetaClient::createEdgeIndex( bool ifNotExists, const cpp2::IndexParams* indexParams, const std::string* comment) { + memory::MemoryCheckOffGuard g; cpp2::CreateEdgeIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(indexName); @@ -1886,6 +1947,7 @@ folly::Future> MetaClient::createEdgeIndex( folly::Future> MetaClient::dropEdgeIndex(GraphSpaceID spaceId, std::string name, bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropEdgeIndexReq req; req.space_id_ref() = spaceId; req.index_name_ref() = std::move(name); @@ -1905,6 +1967,7 @@ folly::Future> MetaClient::dropEdgeIndex(GraphSpaceID spaceId, folly::Future> MetaClient::getEdgeIndex(GraphSpaceID spaceId, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::GetEdgeIndexReq req; req.space_id_ref() = spaceId; req.index_name_ref() = std::move(name); @@ -1921,6 +1984,7 @@ folly::Future> MetaClient::getEdgeIndex(GraphSpaceID s folly::Future>> MetaClient::listEdgeIndexes( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListEdgeIndexesReq req; req.space_id_ref() = spaceId; @@ -1937,6 +2001,7 @@ folly::Future>> MetaClient::listEdgeIndexe } StatusOr MetaClient::getSpaceVidLen(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1956,6 +2021,7 @@ StatusOr MetaClient::getSpaceVidLen(const GraphSpaceID& spaceId) { } StatusOr MetaClient::getSpaceVidType(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1978,6 +2044,7 @@ StatusOr MetaClient::getSpaceVidType(const GraphSpac } StatusOr MetaClient::getSpaceDesc(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1992,6 +2059,7 @@ StatusOr MetaClient::getSpaceDesc(const GraphSpaceID& spaceId) } StatusOr MetaClient::getIsolationLevel(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; auto spaceDescStatus = getSpaceDesc(spaceId); if (!spaceDescStatus.ok()) { return spaceDescStatus.status(); @@ -2002,6 +2070,7 @@ StatusOr MetaClient::getIsolationLevel(GraphSpaceID StatusOr> MetaClient::getTagSchemaFromCache( GraphSpaceID spaceId, TagID tagID, SchemaVer ver) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2023,6 +2092,7 @@ StatusOr> MetaClient::getTagSchemaFr StatusOr> MetaClient::getEdgeSchemaFromCache( GraphSpaceID spaceId, EdgeType edgeType, SchemaVer ver) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2043,6 +2113,7 @@ StatusOr> MetaClient::getEdgeSchemaF } StatusOr MetaClient::getAllVerTagSchema(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2056,6 +2127,7 @@ StatusOr MetaClient::getAllVerTagSchema(GraphSpaceID spaceId) { } StatusOr MetaClient::getAllLatestVerTagSchema(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2075,6 +2147,7 @@ StatusOr MetaClient::getAllLatestVerTagSchema(const GraphSpaceID& spa } StatusOr MetaClient::getAllVerEdgeSchema(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2088,6 +2161,7 @@ StatusOr MetaClient::getAllVerEdgeSchema(GraphSpaceID spaceId) { } StatusOr MetaClient::getAllLatestVerEdgeSchemaFromCache(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2107,6 +2181,7 @@ StatusOr MetaClient::getAllLatestVerEdgeSchemaFromCache(const GraphS } folly::Future> MetaClient::rebuildEdgeIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::RebuildIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(name); @@ -2125,6 +2200,7 @@ folly::Future> MetaClient::rebuildEdgeIndex(GraphSpaceID spaceID, folly::Future>> MetaClient::listEdgeIndexStatus( GraphSpaceID spaceID) { + memory::MemoryCheckOffGuard g; cpp2::ListIndexStatusReq req; req.space_id_ref() = spaceID; @@ -2142,6 +2218,7 @@ folly::Future>> MetaClient::listEdgeInde StatusOr> MetaClient::getTagIndexByNameFromCache( const GraphSpaceID space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2160,6 +2237,7 @@ StatusOr> MetaClient::getTagIndexByNameFromCach StatusOr> MetaClient::getEdgeIndexByNameFromCache( const GraphSpaceID space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2178,6 +2256,7 @@ StatusOr> MetaClient::getEdgeIndexByNameFromCac StatusOr> MetaClient::getTagIndexFromCache(GraphSpaceID spaceId, IndexID indexID) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2201,6 +2280,7 @@ StatusOr> MetaClient::getTagIndexFromCache(Grap StatusOr MetaClient::getRelatedTagIDByIndexNameFromCache(const GraphSpaceID space, const std::string& indexName) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2216,6 +2296,7 @@ StatusOr MetaClient::getRelatedTagIDByIndexNameFromCache(const GraphSpace StatusOr> MetaClient::getEdgeIndexFromCache(GraphSpaceID spaceId, IndexID indexId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2239,6 +2320,7 @@ StatusOr> MetaClient::getEdgeIndexFromCache(Gra StatusOr MetaClient::getRelatedEdgeTypeByIndexNameFromCache( const GraphSpaceID space, const std::string& indexName) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2254,6 +2336,7 @@ StatusOr MetaClient::getRelatedEdgeTypeByIndexNameFromCache( StatusOr>> MetaClient::getTagIndexesFromCache( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2278,6 +2361,7 @@ StatusOr>> MetaClient::getTagIndexe StatusOr>> MetaClient::getEdgeIndexesFromCache( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2301,6 +2385,7 @@ StatusOr>> MetaClient::getEdgeIndex } StatusOr MetaClient::getStorageLeaderFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2332,18 +2417,21 @@ StatusOr MetaClient::getStorageLeaderFromCache(GraphSpaceID spaceId, P void MetaClient::updateStorageLeader(GraphSpaceID spaceId, PartitionID partId, const HostAddr& leader) { + memory::MemoryCheckOffGuard g; VLOG(1) << "Update the leader for [" << spaceId << ", " << partId << "] to " << leader; folly::SharedMutex::WriteHolder holder(leadersLock_); leadersInfo_.leaderMap_[{spaceId, partId}] = leader; } void MetaClient::invalidStorageLeader(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; VLOG(1) << "Invalidate the leader for [" << spaceId << ", " << partId << "]"; folly::SharedMutex::WriteHolder holder(leadersLock_); leadersInfo_.leaderMap_.erase({spaceId, partId}); } StatusOr MetaClient::getLeaderInfo() { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2356,6 +2444,7 @@ const std::vector& MetaClient::getAddresses() { } std::vector MetaClient::getRolesByUserFromCache(const std::string& user) { + memory::MemoryCheckOffGuard g; if (!ready_) { return std::vector(0); } @@ -2369,6 +2458,7 @@ std::vector MetaClient::getRolesByUserFromCache(const std::strin } Status MetaClient::authCheckFromCache(const std::string& account, const std::string& password) { + memory::MemoryCheckOffGuard g; // Check meta service status if (!ready_) { return Status::Error("Meta Service not ready"); @@ -2437,6 +2527,7 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str } bool MetaClient::checkShadowAccountFromCache(const std::string& account) { + memory::MemoryCheckOffGuard g; if (!ready_) { return false; } @@ -2450,6 +2541,7 @@ bool MetaClient::checkShadowAccountFromCache(const std::string& account) { } StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto spaceInfo = metadata.localCache_.find(spaceId); @@ -2466,6 +2558,7 @@ StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID } StatusOr> MetaClient::getStorageHosts() { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2477,6 +2570,7 @@ StatusOr> MetaClient::getStorageHosts() { StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& space, const TagID& tagId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2491,6 +2585,7 @@ StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& StatusOr MetaClient::getLatestEdgeVersionFromCache(const GraphSpaceID& space, const EdgeType& edgeType) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2504,6 +2599,7 @@ StatusOr MetaClient::getLatestEdgeVersionFromCache(const GraphSpaceID } folly::Future> MetaClient::heartbeat() { + memory::MemoryCheckOffGuard g; cpp2::HBReq req; req.host_ref() = options_.localHost_; req.role_ref() = options_.role_; @@ -2596,6 +2692,7 @@ folly::Future> MetaClient::heartbeat() { folly::Future> MetaClient::createUser(std::string account, std::string password, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateUserReq req; req.account_ref() = std::move(account); req.encoded_pwd_ref() = std::move(password); @@ -2613,6 +2710,7 @@ folly::Future> MetaClient::createUser(std::string account, } folly::Future> MetaClient::dropUser(std::string account, bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropUserReq req; req.account_ref() = std::move(account); req.if_exists_ref() = ifExists; @@ -2629,6 +2727,7 @@ folly::Future> MetaClient::dropUser(std::string account, bool ifE } folly::Future> MetaClient::alterUser(std::string account, std::string password) { + memory::MemoryCheckOffGuard g; cpp2::AlterUserReq req; req.account_ref() = std::move(account); req.encoded_pwd_ref() = std::move(password); @@ -2645,6 +2744,7 @@ folly::Future> MetaClient::alterUser(std::string account, std::st } folly::Future> MetaClient::grantToUser(cpp2::RoleItem roleItem) { + memory::MemoryCheckOffGuard g; cpp2::GrantRoleReq req; req.role_item_ref() = std::move(roleItem); folly::Promise> promise; @@ -2660,6 +2760,7 @@ folly::Future> MetaClient::grantToUser(cpp2::RoleItem roleItem) { } folly::Future> MetaClient::revokeFromUser(cpp2::RoleItem roleItem) { + memory::MemoryCheckOffGuard g; cpp2::RevokeRoleReq req; req.role_item_ref() = std::move(roleItem); folly::Promise> promise; @@ -2675,6 +2776,7 @@ folly::Future> MetaClient::revokeFromUser(cpp2::RoleItem roleItem } folly::Future>> MetaClient::listUsers() { + memory::MemoryCheckOffGuard g; cpp2::ListUsersReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -2687,6 +2789,7 @@ folly::Future>> MetaClient } folly::Future>> MetaClient::listRoles(GraphSpaceID space) { + memory::MemoryCheckOffGuard g; cpp2::ListRolesReq req; req.space_id_ref() = std::move(space); folly::Promise>> promise; @@ -2702,6 +2805,7 @@ folly::Future>> MetaClient::listRoles(Graph folly::Future> MetaClient::changePassword(std::string account, std::string newPwd, std::string oldPwd) { + memory::MemoryCheckOffGuard g; cpp2::ChangePasswordReq req; req.account_ref() = std::move(account); req.new_encoded_pwd_ref() = std::move(newPwd); @@ -2719,6 +2823,7 @@ folly::Future> MetaClient::changePassword(std::string account, } folly::Future>> MetaClient::getUserRoles(std::string account) { + memory::MemoryCheckOffGuard g; cpp2::GetUserRolesReq req; req.account_ref() = std::move(account); folly::Promise>> promise; @@ -2732,6 +2837,7 @@ folly::Future>> MetaClient::getUserRoles(st } folly::Future> MetaClient::regConfig(const std::vector& items) { + memory::MemoryCheckOffGuard g; cpp2::RegConfigReq req; req.items_ref() = items; folly::Promise> promise; @@ -2748,6 +2854,7 @@ folly::Future> MetaClient::regConfig(const std::vector>> MetaClient::getConfig( const cpp2::ConfigModule& module, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!configReady_) { return Status::Error("Not ready!"); } @@ -2769,6 +2876,7 @@ folly::Future>> MetaClient::getConfig( folly::Future> MetaClient::setConfig(const cpp2::ConfigModule& module, const std::string& name, const Value& value) { + memory::MemoryCheckOffGuard g; cpp2::ConfigItem item; item.module_ref() = module; item.name_ref() = name; @@ -2790,6 +2898,7 @@ folly::Future> MetaClient::setConfig(const cpp2::ConfigModule& mo folly::Future>> MetaClient::listConfigs( const cpp2::ConfigModule& module) { + memory::MemoryCheckOffGuard g; cpp2::ListConfigsReq req; req.module_ref() = module; folly::Promise>> promise; @@ -2803,6 +2912,7 @@ folly::Future>> MetaClient::listConfigs( } folly::Future> MetaClient::createSnapshot() { + memory::MemoryCheckOffGuard g; cpp2::CreateSnapshotReq req; folly::Promise> promise; auto future = promise.getFuture(); @@ -2820,6 +2930,7 @@ folly::Future> MetaClient::createSnapshot() { } folly::Future> MetaClient::dropSnapshot(const std::string& name) { + memory::MemoryCheckOffGuard g; cpp2::DropSnapshotReq req; std::vector names{name}; req.names_ref() = names; @@ -2836,6 +2947,7 @@ folly::Future> MetaClient::dropSnapshot(const std::string& name) } folly::Future>> MetaClient::listSnapshots() { + memory::MemoryCheckOffGuard g; cpp2::ListSnapshotsReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -2852,6 +2964,7 @@ folly::Future>> MetaClient::listSnapshots() folly::Future> MetaClient::addListener(GraphSpaceID spaceId, cpp2::ListenerType type, std::vector hosts) { + memory::MemoryCheckOffGuard g; cpp2::AddListenerReq req; req.space_id_ref() = spaceId; req.type_ref() = type; @@ -2870,6 +2983,7 @@ folly::Future> MetaClient::addListener(GraphSpaceID spaceId, folly::Future> MetaClient::removeListener(GraphSpaceID spaceId, cpp2::ListenerType type) { + memory::MemoryCheckOffGuard g; cpp2::RemoveListenerReq req; req.space_id_ref() = spaceId; req.type_ref() = type; @@ -2887,6 +3001,7 @@ folly::Future> MetaClient::removeListener(GraphSpaceID spaceId, folly::Future>> MetaClient::listListener( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListListenerReq req; req.space_id_ref() = spaceId; folly::Promise>> promise; @@ -2902,6 +3017,7 @@ folly::Future>> MetaClient::listListene } bool MetaClient::registerCfg() { + memory::MemoryCheckOffGuard g; auto ret = regConfig(gflagsDeclared_).get(); if (ret.ok()) { LOG(INFO) << "Register gflags ok " << gflagsDeclared_.size(); @@ -2912,6 +3028,7 @@ bool MetaClient::registerCfg() { StatusOr>> MetaClient::getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAddr& host) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2932,6 +3049,7 @@ MetaClient::getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAdd } StatusOr MetaClient::getListenersByHostFromCache(const HostAddr& host) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2941,6 +3059,7 @@ StatusOr MetaClient::getListenersByHostFromCache(const HostAddr& h } ListenersMap MetaClient::doGetListenersMap(const HostAddr& host, const LocalCache& localCache) { + memory::MemoryCheckOffGuard g; ListenersMap listenersMap; for (const auto& space : localCache) { auto spaceId = space.first; @@ -2970,6 +3089,7 @@ ListenersMap MetaClient::doGetListenersMap(const HostAddr& host, const LocalCach StatusOr MetaClient::getListenerHostsBySpacePartType(GraphSpaceID spaceId, PartitionID partId, cpp2::ListenerType type) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2992,6 +3112,7 @@ StatusOr MetaClient::getListenerHostsBySpacePartType(GraphSpaceID spac StatusOr> MetaClient::getListenerHostTypeBySpacePartType( GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3017,6 +3138,7 @@ StatusOr> MetaClient::getListenerHostTypeBySpace } bool MetaClient::loadCfg() { + memory::MemoryCheckOffGuard g; // UNKNOWN role will skip heartbeat if (options_.skipConfig_ || (options_.role_ != cpp2::HostRole::UNKNOWN && localCfgLastUpdateTime_ == metadLastUpdateTime_)) { @@ -3057,6 +3179,7 @@ bool MetaClient::loadCfg() { } void MetaClient::updateGflagsValue(const cpp2::ConfigItem& item) { + memory::MemoryCheckOffGuard g; if (item.get_mode() != cpp2::ConfigMode::MUTABLE) { return; } @@ -3085,6 +3208,7 @@ void MetaClient::updateGflagsValue(const cpp2::ConfigItem& item) { } void MetaClient::updateNestedGflags(const std::unordered_map& nameValues) { + memory::MemoryCheckOffGuard g; std::unordered_map optionMap; for (const auto& value : nameValues) { optionMap.emplace(value.first, value.second.toString()); @@ -3098,12 +3222,14 @@ void MetaClient::updateNestedGflags(const std::unordered_map } Status MetaClient::refreshCache() { + memory::MemoryCheckOffGuard g; auto ret = bgThread_->addTask(&MetaClient::loadData, this).get(); return ret ? Status::OK() : Status::Error("Load data failed"); } void MetaClient::loadLeader(const std::vector& hostItems, const SpaceNameIdMap& spaceIndexByName) { + memory::MemoryCheckOffGuard g; LeaderInfo leaderInfo; for (auto& item : hostItems) { for (auto& spaceEntry : item.get_leader_parts()) { @@ -3143,6 +3269,7 @@ void MetaClient::loadLeader(const std::vector& hostItems, } folly::Future> MetaClient::addHosts(std::vector hosts) { + memory::MemoryCheckOffGuard g; cpp2::AddHostsReq req; req.hosts_ref() = std::move(hosts); @@ -3159,6 +3286,7 @@ folly::Future> MetaClient::addHosts(std::vector hosts) } folly::Future> MetaClient::dropHosts(std::vector hosts) { + memory::MemoryCheckOffGuard g; cpp2::DropHostsReq req; req.hosts_ref() = std::move(hosts); @@ -3176,6 +3304,7 @@ folly::Future> MetaClient::dropHosts(std::vector hosts) folly::Future> MetaClient::mergeZone(std::vector zones, std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::MergeZoneReq req; req.zone_name_ref() = std::move(zoneName); req.zones_ref() = std::move(zones); @@ -3193,6 +3322,7 @@ folly::Future> MetaClient::mergeZone(std::vector zon folly::Future> MetaClient::divideZone( std::string zoneName, std::unordered_map> zoneItems) { + memory::MemoryCheckOffGuard g; cpp2::DivideZoneReq req; req.zone_name_ref() = std::move(zoneName); req.zone_items_ref() = std::move(zoneItems); @@ -3210,6 +3340,7 @@ folly::Future> MetaClient::divideZone( folly::Future> MetaClient::renameZone(std::string originalZoneName, std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::RenameZoneReq req; req.original_zone_name_ref() = std::move(originalZoneName); req.zone_name_ref() = std::move(zoneName); @@ -3226,6 +3357,7 @@ folly::Future> MetaClient::renameZone(std::string originalZoneNam } folly::Future> MetaClient::dropZone(std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::DropZoneReq req; req.zone_name_ref() = std::move(zoneName); @@ -3244,6 +3376,7 @@ folly::Future> MetaClient::dropZone(std::string zoneName) { folly::Future> MetaClient::addHostsIntoZone(std::vector hosts, std::string zoneName, bool isNew) { + memory::MemoryCheckOffGuard g; cpp2::AddHostsIntoZoneReq req; req.hosts_ref() = hosts; req.zone_name_ref() = zoneName; @@ -3262,6 +3395,7 @@ folly::Future> MetaClient::addHostsIntoZone(std::vector } folly::Future>> MetaClient::getZone(std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::GetZoneReq req; req.zone_name_ref() = std::move(zoneName); @@ -3276,6 +3410,7 @@ folly::Future>> MetaClient::getZone(std::string z } folly::Future>> MetaClient::listZones() { + memory::MemoryCheckOffGuard g; cpp2::ListZonesReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -3288,6 +3423,7 @@ folly::Future>> MetaClient::listZones() { } folly::Future> MetaClient::getStats(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::GetStatsReq req; req.space_id_ref() = (spaceId); folly::Promise> promise; @@ -3307,6 +3443,7 @@ folly::Future> MetaClient::reportTaskFinish( int32_t taskId, nebula::cpp2::ErrorCode taskErrCode, cpp2::StatsItem* statisticItem) { + memory::MemoryCheckOffGuard g; cpp2::ReportTaskReq req; req.code_ref() = taskErrCode; req.space_id_ref() = spaceId; @@ -3328,6 +3465,7 @@ folly::Future> MetaClient::reportTaskFinish( folly::Future> MetaClient::signInService( const cpp2::ExternalServiceType& type, const std::vector& clients) { + memory::MemoryCheckOffGuard g; cpp2::SignInServiceReq req; req.type_ref() = type; req.clients_ref() = clients; @@ -3345,6 +3483,7 @@ folly::Future> MetaClient::signInService( } folly::Future> MetaClient::signOutService(const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; cpp2::SignOutServiceReq req; req.type_ref() = type; folly::Promise> promise; @@ -3362,6 +3501,7 @@ folly::Future> MetaClient::signOutService(const cpp2::ExternalSer folly::Future> MetaClient::listServiceClients( const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; cpp2::ListServiceClientsReq req; req.type_ref() = type; folly::Promise> promise; @@ -3378,6 +3518,7 @@ folly::Future> MetaClient::listServiceClients( StatusOr> MetaClient::getServiceClientsFromCache( const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3394,6 +3535,7 @@ StatusOr> MetaClient::getServiceClientsFromCach folly::Future> MetaClient::createFTIndex(const std::string& name, const cpp2::FTIndex& index) { + memory::MemoryCheckOffGuard g; cpp2::CreateFTIndexReq req; req.fulltext_index_name_ref() = name; req.index_ref() = index; @@ -3412,6 +3554,7 @@ folly::Future> MetaClient::createFTIndex(const std::string& name, folly::Future> MetaClient::dropFTIndex(GraphSpaceID spaceId, const std::string& name) { + memory::MemoryCheckOffGuard g; cpp2::DropFTIndexReq req; req.fulltext_index_name_ref() = name; req.space_id_ref() = spaceId; @@ -3430,6 +3573,7 @@ folly::Future> MetaClient::dropFTIndex(GraphSpaceID spaceId, folly::Future>> MetaClient::listFTIndexes() { + memory::MemoryCheckOffGuard g; cpp2::ListFTIndexesReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -3444,6 +3588,7 @@ MetaClient::listFTIndexes() { } StatusOr> MetaClient::getFTIndexesFromCache() { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3454,6 +3599,7 @@ StatusOr> MetaClient::getFTIndexe StatusOr> MetaClient::getFTIndexBySpaceFromCache( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3470,6 +3616,7 @@ StatusOr> MetaClient::getFTIndexB StatusOr> MetaClient::getFTIndexFromCache( GraphSpaceID spaceId, int32_t schemaId, const std::string& field) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3490,6 +3637,7 @@ StatusOr> MetaClient::getFTIndexFromCache( StatusOr> MetaClient::getFTIndexFromCache( GraphSpaceID spaceId, int32_t schemaId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3509,6 +3657,7 @@ StatusOr> MetaClient::getFTIndexF StatusOr MetaClient::getFTIndexByNameFromCache(GraphSpaceID spaceId, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3523,6 +3672,7 @@ StatusOr MetaClient::getFTIndexByNameFromCache(GraphSpaceID space folly::Future> MetaClient::createSession( const std::string& userName, const HostAddr& graphAddr, const std::string& clientIp) { + memory::MemoryCheckOffGuard g; cpp2::CreateSessionReq req; req.user_ref() = userName; req.graph_addr_ref() = graphAddr; @@ -3540,6 +3690,7 @@ folly::Future> MetaClient::createSession( folly::Future> MetaClient::updateSessions( const std::vector& sessions) { + memory::MemoryCheckOffGuard g; cpp2::UpdateSessionsReq req; req.sessions_ref() = sessions; folly::Promise> promise; @@ -3554,6 +3705,7 @@ folly::Future> MetaClient::updateSessions( } folly::Future> MetaClient::listSessions() { + memory::MemoryCheckOffGuard g; cpp2::ListSessionsReq req; folly::Promise> promise; auto future = promise.getFuture(); @@ -3566,6 +3718,7 @@ folly::Future> MetaClient::listSessions() { } folly::Future> MetaClient::getSession(SessionID sessionId) { + memory::MemoryCheckOffGuard g; cpp2::GetSessionReq req; req.session_id_ref() = sessionId; folly::Promise> promise; @@ -3580,6 +3733,7 @@ folly::Future> MetaClient::getSession(SessionID s folly::Future> MetaClient::removeSessions( const std::vector& sessionIds) { + memory::MemoryCheckOffGuard g; cpp2::RemoveSessionReq req; req.session_ids_ref() = sessionIds; folly::Promise> promise; @@ -3595,6 +3749,7 @@ folly::Future> MetaClient::removeSessions( folly::Future> MetaClient::killQuery( std::unordered_map> killQueries) { + memory::MemoryCheckOffGuard g; cpp2::KillQueryReq req; req.kill_queries_ref() = std::move(killQueries); folly::Promise> promise; @@ -3609,6 +3764,7 @@ folly::Future> MetaClient::killQuery( } folly::Future> MetaClient::getWorkerId(std::string ipAddr) { + memory::MemoryCheckOffGuard g; cpp2::GetWorkerIdReq req; req.host_ref() = std::move(ipAddr); @@ -3624,6 +3780,7 @@ folly::Future> MetaClient::getWorkerId(std::string ipAddr) { } folly::Future> MetaClient::getSegmentId(int64_t length) { + memory::MemoryCheckOffGuard g; auto req = cpp2::GetSegmentIdReq(); req.length_ref() = length; @@ -3639,6 +3796,7 @@ folly::Future> MetaClient::getSegmentId(int64_t length) { } bool MetaClient::loadSessions() { + memory::MemoryCheckOffGuard g; auto session_list = listSessions().get(); if (!session_list.ok()) { LOG(ERROR) << "List sessions failed, status:" << session_list.status(); @@ -3658,6 +3816,7 @@ bool MetaClient::loadSessions() { } StatusOr MetaClient::getSessionFromCache(const nebula::SessionID& session_id) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3671,6 +3830,7 @@ StatusOr MetaClient::getSessionFromCache(const nebula::SessionID& } bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) { + memory::MemoryCheckOffGuard g; static thread_local int check_counter = 0; // Inaccurate in a multi-threaded environment, but it is not important check_counter = (check_counter + 1) & ((1 << FLAGS_check_plan_killed_frequency) - 1); @@ -3682,6 +3842,7 @@ bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) } Status MetaClient::verifyVersion() { + memory::MemoryCheckOffGuard g; auto req = cpp2::VerifyClientVersionReq(); req.build_version_ref() = getOriginVersion(); req.host_ref() = options_.localHost_; @@ -3705,6 +3866,7 @@ Status MetaClient::verifyVersion() { } Status MetaClient::saveVersionToMeta() { + memory::MemoryCheckOffGuard g; auto req = cpp2::SaveGraphVersionReq(); req.build_version_ref() = getOriginVersion(); req.host_ref() = options_.localHost_; diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h index beb8d25d958..c9b530595a5 100644 --- a/src/clients/storage/StorageClientBase-inl.h +++ b/src/clients/storage/StorageClientBase-inl.h @@ -76,6 +76,7 @@ StorageClientBase::collectResponse( folly::EventBase* evb, std::unordered_map requests, RemoteFunc&& remoteFunc) { + memory::MemoryCheckOffGuard offGuard; std::vector>> respFutures; respFutures.reserve(requests.size()); diff --git a/src/codec/RowReader.cpp b/src/codec/RowReader.cpp index fa5c190dcbf..ff365f0970d 100644 --- a/src/codec/RowReader.cpp +++ b/src/codec/RowReader.cpp @@ -15,7 +15,7 @@ namespace nebula { * class RowReader::Cell * ********************************************/ -Value RowReader::Cell::value() const noexcept { +Value RowReader::Cell::value() const { return iter_->reader_->getValueByIndex(iter_->index_); } @@ -50,7 +50,7 @@ RowReader::Iterator& RowReader::Iterator::operator++() { * ********************************************/ -bool RowReader::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept { +bool RowReader::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) { schema_ = schema; data_ = row; diff --git a/src/codec/RowReader.h b/src/codec/RowReader.h index 91f09896a39..364f5dae57b 100644 --- a/src/codec/RowReader.h +++ b/src/codec/RowReader.h @@ -25,7 +25,7 @@ class RowReader { friend class Iterator; public: - Value value() const noexcept; + Value value() const; private: const Iterator* iter_; @@ -74,7 +74,7 @@ class RowReader { * @param prop Property name * @return Value Property value */ - virtual Value getValueByName(const std::string& prop) const noexcept = 0; + virtual Value getValueByName(const std::string& prop) const = 0; /** * @brief Get the property value by index in schema @@ -82,7 +82,7 @@ class RowReader { * @param index Index in Schema * @return Value Property value */ - virtual Value getValueByIndex(const int64_t index) const noexcept = 0; + virtual Value getValueByIndex(const int64_t index) const = 0; /** * @brief Get the timestamp in value @@ -108,7 +108,7 @@ class RowReader { * * @return Iterator */ - virtual Iterator begin() const noexcept { + virtual Iterator begin() const { return Iterator(this, 0); } @@ -117,7 +117,7 @@ class RowReader { * * @return const Iterator& */ - virtual const Iterator& end() const noexcept { + virtual const Iterator& end() const { return endIter_; } @@ -170,7 +170,7 @@ class RowReader { * @param row * @return Whether reset succeed */ - virtual bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept; + virtual bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row); private: Iterator endIter_; diff --git a/src/codec/RowReaderV1.cpp b/src/codec/RowReaderV1.cpp index a8f9023c258..f39af174003 100644 --- a/src/codec/RowReaderV1.cpp +++ b/src/codec/RowReaderV1.cpp @@ -27,7 +27,7 @@ using nebula::cpp2::PropertyType; * class RowReaderV1 * ********************************************/ -bool RowReaderV1::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept { +bool RowReaderV1::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) { RowReader::resetImpl(schema, row); DCHECK(schema_ != nullptr) << "A schema must be provided"; @@ -95,7 +95,7 @@ bool RowReaderV1::processHeader(folly::StringPiece row) { return true; } -int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept { +int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const { const PropertyType& vType = getSchema()->getFieldType(index); if (offsets_[index + 1] >= 0) { return offsets_[index + 1]; @@ -160,7 +160,7 @@ int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept { return offset; } -int64_t RowReaderV1::skipToField(int64_t index) const noexcept { +int64_t RowReaderV1::skipToField(int64_t index) const { DCHECK_GE(index, 0); if (index >= static_cast(schema_->getNumFields())) { // Index is out of range @@ -191,12 +191,12 @@ int64_t RowReaderV1::skipToField(int64_t index) const noexcept { * Get the property value * ***********************************************************/ -Value RowReaderV1::getValueByName(const std::string& prop) const noexcept { +Value RowReaderV1::getValueByName(const std::string& prop) const { int64_t index = getSchema()->getFieldIndex(prop); return getValueByIndex(index); } -Value RowReaderV1::getValueByIndex(const int64_t index) const noexcept { +Value RowReaderV1::getValueByIndex(const int64_t index) const { if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return Value(NullType::UNKNOWN_PROP); } @@ -233,7 +233,7 @@ int64_t RowReaderV1::getTimestamp() const noexcept { * Get the property value from the serialized binary string * ***********************************************************/ -Value RowReaderV1::getBool(int64_t index) const noexcept { +Value RowReaderV1::getBool(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -272,7 +272,7 @@ Value RowReaderV1::getBool(int64_t index) const noexcept { return v; } -Value RowReaderV1::getInt(int64_t index) const noexcept { +Value RowReaderV1::getInt(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -296,7 +296,7 @@ Value RowReaderV1::getInt(int64_t index) const noexcept { return v; } -Value RowReaderV1::getFloat(int64_t index) const noexcept { +Value RowReaderV1::getFloat(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -334,7 +334,7 @@ Value RowReaderV1::getFloat(int64_t index) const noexcept { return v; } -Value RowReaderV1::getDouble(int64_t index) const noexcept { +Value RowReaderV1::getDouble(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -368,7 +368,7 @@ Value RowReaderV1::getDouble(int64_t index) const noexcept { return v; } -Value RowReaderV1::getString(int64_t index) const noexcept { +Value RowReaderV1::getString(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -391,7 +391,7 @@ Value RowReaderV1::getString(int64_t index) const noexcept { return v; } -Value RowReaderV1::getInt64(int64_t index) const noexcept { +Value RowReaderV1::getInt64(int64_t index) const { RR_GET_OFFSET() Value v; int64_t val; @@ -425,7 +425,7 @@ Value RowReaderV1::getInt64(int64_t index) const noexcept { return v; } -Value RowReaderV1::getVid(int64_t index) const noexcept { +Value RowReaderV1::getVid(int64_t index) const { auto fieldType = getSchema()->getFieldType(index); if (fieldType == PropertyType::INT64 || fieldType == PropertyType::VID) { // Since 2.0, vid has been defined as a binary array. So we need to convert @@ -445,7 +445,7 @@ Value RowReaderV1::getVid(int64_t index) const noexcept { * Low-level functions to read from the bytes * ***********************************************************/ -int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const noexcept { +int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const { const uint8_t* start = reinterpret_cast(&(buffer_[offset])); folly::ByteRange range(start, buffer_.size() - offset); @@ -457,7 +457,7 @@ int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const noexcept { return range.begin() - start; } -int32_t RowReaderV1::readFloat(int64_t offset, float& v) const noexcept { +int32_t RowReaderV1::readFloat(int64_t offset, float& v) const { if (offset + sizeof(float) > buffer_.size()) { return -1; } @@ -467,7 +467,7 @@ int32_t RowReaderV1::readFloat(int64_t offset, float& v) const noexcept { return sizeof(float); } -int32_t RowReaderV1::readDouble(int64_t offset, double& v) const noexcept { +int32_t RowReaderV1::readDouble(int64_t offset, double& v) const { if (offset + sizeof(double) > buffer_.size()) { return -1; } @@ -477,7 +477,7 @@ int32_t RowReaderV1::readDouble(int64_t offset, double& v) const noexcept { return sizeof(double); } -int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const noexcept { +int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const { int64_t strLen = 0; int32_t intLen = readInteger(offset, strLen); CHECK_GT(intLen, 0) << "Invalid string length"; @@ -489,7 +489,7 @@ int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const noe return intLen + strLen; } -int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const noexcept { +int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const { if (offset + sizeof(int64_t) > buffer_.size()) { return -1; } @@ -500,7 +500,7 @@ int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const noexcept { return sizeof(int64_t); } -int32_t RowReaderV1::readVid(int64_t offset, int64_t& v) const noexcept { +int32_t RowReaderV1::readVid(int64_t offset, int64_t& v) const { return readInt64(offset, v); } diff --git a/src/codec/RowReaderV1.h b/src/codec/RowReaderV1.h index 473e338fdbe..5197f3e5761 100644 --- a/src/codec/RowReaderV1.h +++ b/src/codec/RowReaderV1.h @@ -27,8 +27,8 @@ class RowReaderV1 : public RowReader { public: ~RowReaderV1() = default; - Value getValueByName(const std::string& prop) const noexcept override; - Value getValueByIndex(const int64_t index) const noexcept override; + Value getValueByName(const std::string& prop) const override; + Value getValueByIndex(const int64_t index) const override; int64_t getTimestamp() const noexcept override; int32_t readerVer() const noexcept override { @@ -40,7 +40,7 @@ class RowReaderV1 : public RowReader { } protected: - bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept override; + bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) override; private: int32_t headerLen_ = 0; @@ -72,31 +72,31 @@ class RowReaderV1 : public RowReader { // When succeeded, the method returns the offset pointing to the // next field // When failed, the method returns a negative number - int64_t skipToNext(int64_t index, int64_t offset) const noexcept; + int64_t skipToNext(int64_t index, int64_t offset) const; // Skip to the {index}Th field // The method returns the offset of the field // It returns a negative number when the data corrupts - int64_t skipToField(int64_t index) const noexcept; + int64_t skipToField(int64_t index) const; // Following methods assume the parameters index are valid // When succeeded, offset will advance - Value getBool(int64_t index) const noexcept; - Value getInt(int64_t index) const noexcept; - Value getFloat(int64_t index) const noexcept; - Value getDouble(int64_t index) const noexcept; - Value getString(int64_t index) const noexcept; - Value getInt64(int64_t index) const noexcept; - Value getVid(int64_t index) const noexcept; + Value getBool(int64_t index) const; + Value getInt(int64_t index) const; + Value getFloat(int64_t index) const; + Value getDouble(int64_t index) const; + Value getString(int64_t index) const; + Value getInt64(int64_t index) const; + Value getVid(int64_t index) const; // The following methods all return the number of bytes read // A negative number will be returned if an error occurs - int32_t readInteger(int64_t offset, int64_t& v) const noexcept; - int32_t readFloat(int64_t offset, float& v) const noexcept; - int32_t readDouble(int64_t offset, double& v) const noexcept; - int32_t readString(int64_t offset, folly::StringPiece& v) const noexcept; - int32_t readInt64(int64_t offset, int64_t& v) const noexcept; - int32_t readVid(int64_t offset, int64_t& v) const noexcept; + int32_t readInteger(int64_t offset, int64_t& v) const; + int32_t readFloat(int64_t offset, float& v) const; + int32_t readDouble(int64_t offset, double& v) const; + int32_t readString(int64_t offset, folly::StringPiece& v) const; + int32_t readInt64(int64_t offset, int64_t& v) const; + int32_t readVid(int64_t offset, int64_t& v) const; }; } // namespace nebula diff --git a/src/codec/RowReaderV2.cpp b/src/codec/RowReaderV2.cpp index 384a171494f..9e662a27677 100644 --- a/src/codec/RowReaderV2.cpp +++ b/src/codec/RowReaderV2.cpp @@ -9,7 +9,7 @@ namespace nebula { using nebula::cpp2::PropertyType; -bool RowReaderV2::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept { +bool RowReaderV2::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) { RowReader::resetImpl(schema, row); DCHECK(!!schema_); @@ -45,12 +45,12 @@ bool RowReaderV2::isNull(size_t pos) const { return flag != 0; } -Value RowReaderV2::getValueByName(const std::string& prop) const noexcept { +Value RowReaderV2::getValueByName(const std::string& prop) const { int64_t index = schema_->getFieldIndex(prop); return getValueByIndex(index); } -Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept { +Value RowReaderV2::getValueByIndex(const int64_t index) const { if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return Value(NullType::UNKNOWN_PROP); } diff --git a/src/codec/RowReaderV2.h b/src/codec/RowReaderV2.h index 9bad0bf7f05..9003793e1e9 100644 --- a/src/codec/RowReaderV2.h +++ b/src/codec/RowReaderV2.h @@ -28,8 +28,8 @@ class RowReaderV2 : public RowReader { public: ~RowReaderV2() override = default; - Value getValueByName(const std::string& prop) const noexcept override; - Value getValueByIndex(const int64_t index) const noexcept override; + Value getValueByName(const std::string& prop) const override; + Value getValueByIndex(const int64_t index) const override; int64_t getTimestamp() const noexcept override; int32_t readerVer() const noexcept override { @@ -41,7 +41,7 @@ class RowReaderV2 : public RowReader { } protected: - bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept override; + bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) override; private: size_t headerLen_; diff --git a/src/codec/RowReaderWrapper.cpp b/src/codec/RowReaderWrapper.cpp index 5a100fabcfb..0a1b818c066 100644 --- a/src/codec/RowReaderWrapper.cpp +++ b/src/codec/RowReaderWrapper.cpp @@ -97,7 +97,7 @@ RowReaderWrapper::RowReaderWrapper(const meta::SchemaProviderIf* schema, bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, folly::StringPiece row, - int32_t readerVer) noexcept { + int32_t readerVer) { CHECK_NOTNULL(schema); readerVer_ = readerVer; if (readerVer_ == 1) { @@ -115,8 +115,7 @@ bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, } } -bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, - folly::StringPiece row) noexcept { +bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, folly::StringPiece row) { currReader_ = nullptr; if (schema == nullptr) { return false; @@ -132,7 +131,7 @@ bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, bool RowReaderWrapper::reset( const std::vector>& schemas, - folly::StringPiece row) noexcept { + folly::StringPiece row) { currReader_ = nullptr; SchemaVer schemaVer; int32_t readerVer; diff --git a/src/codec/RowReaderWrapper.h b/src/codec/RowReaderWrapper.h index a619d6d3490..2181670de42 100644 --- a/src/codec/RowReaderWrapper.h +++ b/src/codec/RowReaderWrapper.h @@ -137,9 +137,7 @@ class RowReaderWrapper : public RowReader { * @param readVer * @return Whether reset succeed */ - bool reset(meta::SchemaProviderIf const* schema, - folly::StringPiece row, - int32_t readVer) noexcept; + bool reset(meta::SchemaProviderIf const* schema, folly::StringPiece row, int32_t readVer); /** * @brief Reset current row reader wrapper to of given schema and data @@ -148,7 +146,7 @@ class RowReaderWrapper : public RowReader { * @param row * @return Whether reset succeed */ - bool reset(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept; + bool reset(meta::SchemaProviderIf const* schema, folly::StringPiece row); /** * @brief Reset current row reader wrapper of given schemas and data, the schemas are stored in @@ -159,14 +157,14 @@ class RowReaderWrapper : public RowReader { * @return Whether reset succeed */ bool reset(const std::vector>& schemas, - folly::StringPiece row) noexcept; + folly::StringPiece row); - Value getValueByName(const std::string& prop) const noexcept override { + Value getValueByName(const std::string& prop) const override { DCHECK(!!currReader_); return currReader_->getValueByName(prop); } - Value getValueByIndex(const int64_t index) const noexcept override { + Value getValueByIndex(const int64_t index) const override { DCHECK(!!currReader_); return currReader_->getValueByIndex(index); } @@ -187,12 +185,12 @@ class RowReaderWrapper : public RowReader { return currReader_->headerLen(); } - Iterator begin() const noexcept override { + Iterator begin() const override { DCHECK(!!currReader_); return currReader_->begin(); } - const Iterator& end() const noexcept override { + const Iterator& end() const override { DCHECK(!!currReader_); return currReader_->end(); } diff --git a/src/codec/RowWriterV2.cpp b/src/codec/RowWriterV2.cpp index 17bc7c3a6d4..5fcd457f30a 100644 --- a/src/codec/RowWriterV2.cpp +++ b/src/codec/RowWriterV2.cpp @@ -147,7 +147,7 @@ RowWriterV2::RowWriterV2(RowReader& reader) : RowWriterV2(reader.getSchema()) { } } -void RowWriterV2::processV2EncodedStr() noexcept { +void RowWriterV2::processV2EncodedStr() { CHECK_EQ(0x08, buf_[0] & 0x18); int32_t verBytes = buf_[0] & 0x07; SchemaVer ver = 0; @@ -172,21 +172,21 @@ void RowWriterV2::processV2EncodedStr() noexcept { isSet_.resize(schema_->getNumFields(), true); } -void RowWriterV2::setNullBit(ssize_t pos) noexcept { +void RowWriterV2::setNullBit(ssize_t pos) { static const uint8_t orBits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; size_t offset = headerLen_ + (pos >> 3); buf_[offset] = buf_[offset] | orBits[pos & 0x0000000000000007L]; } -void RowWriterV2::clearNullBit(ssize_t pos) noexcept { +void RowWriterV2::clearNullBit(ssize_t pos) { static const uint8_t andBits[] = {0x7F, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD, 0xFE}; size_t offset = headerLen_ + (pos >> 3); buf_[offset] = buf_[offset] & andBits[pos & 0x0000000000000007L]; } -bool RowWriterV2::checkNullBit(ssize_t pos) const noexcept { +bool RowWriterV2::checkNullBit(ssize_t pos) const { static const uint8_t bits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; size_t offset = headerLen_ + (pos >> 3); @@ -194,7 +194,7 @@ bool RowWriterV2::checkNullBit(ssize_t pos) const noexcept { return flag != 0; } -WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) noexcept { +WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) { CHECK(!finished_) << "You have called finish()"; if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return WriteResult::UNKNOWN_FIELD; @@ -226,13 +226,13 @@ WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) noexcept { } } -WriteResult RowWriterV2::setValue(const std::string& name, const Value& val) noexcept { +WriteResult RowWriterV2::setValue(const std::string& name, const Value& val) { CHECK(!finished_) << "You have called finish()"; int64_t index = schema_->getFieldIndex(name); return setValue(index, val); } -WriteResult RowWriterV2::setNull(ssize_t index) noexcept { +WriteResult RowWriterV2::setNull(ssize_t index) { CHECK(!finished_) << "You have called finish()"; if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return WriteResult::UNKNOWN_FIELD; @@ -249,13 +249,13 @@ WriteResult RowWriterV2::setNull(ssize_t index) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::setNull(const std::string& name) noexcept { +WriteResult RowWriterV2::setNull(const std::string& name) { CHECK(!finished_) << "You have called finish()"; int64_t index = schema_->getFieldIndex(name); return setNull(index); } -WriteResult RowWriterV2::write(ssize_t index, bool v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, bool v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -285,7 +285,7 @@ WriteResult RowWriterV2::write(ssize_t index, bool v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, float v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, float v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -342,7 +342,7 @@ WriteResult RowWriterV2::write(ssize_t index, float v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, double v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, double v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -401,11 +401,11 @@ WriteResult RowWriterV2::write(ssize_t index, double v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint8_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint8_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int8_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int8_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -452,11 +452,11 @@ WriteResult RowWriterV2::write(ssize_t index, int8_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint16_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint16_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int16_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int16_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -506,11 +506,11 @@ WriteResult RowWriterV2::write(ssize_t index, int16_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint32_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint32_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int32_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int32_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -573,11 +573,11 @@ WriteResult RowWriterV2::write(ssize_t index, int32_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint64_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint64_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int64_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int64_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -643,15 +643,15 @@ WriteResult RowWriterV2::write(ssize_t index, int64_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const std::string& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const std::string& v) { return write(index, folly::StringPiece(v)); } -WriteResult RowWriterV2::write(ssize_t index, const char* v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const char* v) { return write(index, folly::StringPiece(v)); } -WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -701,7 +701,7 @@ WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Date& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Date& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -720,7 +720,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Date& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Time& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Time& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -742,7 +742,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Time& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); int16_t year = v.year; @@ -774,7 +774,7 @@ WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Duration& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Duration& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -797,7 +797,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Duration& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Geography& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Geography& v) { auto field = schema_->field(index); auto geoShape = field->geoShape(); if (geoShape != meta::cpp2::GeoShape::ANY && @@ -808,7 +808,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Geography& v) noexcept { return write(index, folly::StringPiece(wkb)); } -WriteResult RowWriterV2::checkUnsetFields() noexcept { +WriteResult RowWriterV2::checkUnsetFields() { DefaultValueContext expCtx; for (size_t i = 0; i < schema_->getNumFields(); i++) { if (!isSet_[i]) { @@ -875,7 +875,7 @@ WriteResult RowWriterV2::checkUnsetFields() noexcept { return WriteResult::SUCCEEDED; } -std::string RowWriterV2::processOutOfSpace() noexcept { +std::string RowWriterV2::processOutOfSpace() { std::string temp; // Reserve enough space to avoid memory re-allocation temp.reserve(headerLen_ + numNullBytes_ + schema_->size() + approxStrLen_ + sizeof(int64_t)); @@ -919,7 +919,7 @@ std::string RowWriterV2::processOutOfSpace() noexcept { return temp; } -WriteResult RowWriterV2::finish() noexcept { +WriteResult RowWriterV2::finish() { CHECK(!finished_) << "You have called finish()"; // First to check whether all fields are set. If not, to check whether diff --git a/src/codec/RowWriterV2.h b/src/codec/RowWriterV2.h index 6b79c00a0ca..6dca13bbafa 100644 --- a/src/codec/RowWriterV2.h +++ b/src/codec/RowWriterV2.h @@ -118,7 +118,7 @@ class RowWriterV2 { * * @return const std::string& */ - const std::string& getEncodedStr() const noexcept { + const std::string& getEncodedStr() const { CHECK(finished_) << "You need to call finish() first"; return buf_; } @@ -128,7 +128,7 @@ class RowWriterV2 { * * @return std::string */ - std::string moveEncodedStr() noexcept { + std::string moveEncodedStr() { CHECK(finished_) << "You need to call finish() first"; return std::move(buf_); } @@ -138,7 +138,7 @@ class RowWriterV2 { * * @return WriteResult Whether encode succeed */ - WriteResult finish() noexcept; + WriteResult finish(); // Data write /** @@ -150,7 +150,7 @@ class RowWriterV2 { * @return WriteResult */ template - WriteResult set(size_t index, T&& v) noexcept { + WriteResult set(size_t index, T&& v) { CHECK(!finished_) << "You have called finish()"; if (index >= schema_->getNumFields()) { return WriteResult::UNKNOWN_FIELD; @@ -168,7 +168,7 @@ class RowWriterV2 { * @return WriteResult */ template - WriteResult set(const std::string& name, T&& v) noexcept { + WriteResult set(const std::string& name, T&& v) { CHECK(!finished_) << "You have called finish()"; int64_t index = schema_->getFieldIndex(name); if (index >= 0) { @@ -185,7 +185,7 @@ class RowWriterV2 { * @param val * @return WriteResult */ - WriteResult setValue(ssize_t index, const Value& val) noexcept; + WriteResult setValue(ssize_t index, const Value& val); /** * @brief Set the value by index @@ -194,7 +194,7 @@ class RowWriterV2 { * @param val * @return WriteResult */ - WriteResult setValue(const std::string& name, const Value& val) noexcept; + WriteResult setValue(const std::string& name, const Value& val); /** * @brief Set null by index @@ -202,7 +202,7 @@ class RowWriterV2 { * @param index * @return WriteResult */ - WriteResult setNull(ssize_t index) noexcept; + WriteResult setNull(ssize_t index); /** * @brief Set null by property name @@ -210,7 +210,7 @@ class RowWriterV2 { * @param name * @return WriteResult */ - WriteResult setNull(const std::string& name) noexcept; + WriteResult setNull(const std::string& name); private: const meta::SchemaProviderIf* schema_; @@ -229,40 +229,40 @@ class RowWriterV2 { bool outOfSpaceStr_; std::vector strList_; - WriteResult checkUnsetFields() noexcept; - std::string processOutOfSpace() noexcept; + WriteResult checkUnsetFields(); + std::string processOutOfSpace(); - void processV2EncodedStr() noexcept; + void processV2EncodedStr(); - void setNullBit(ssize_t pos) noexcept; - void clearNullBit(ssize_t pos) noexcept; + void setNullBit(ssize_t pos); + void clearNullBit(ssize_t pos); // Return true if the flag at the given position is NULL; // otherwise, return false - bool checkNullBit(ssize_t pos) const noexcept; - - WriteResult write(ssize_t index, bool v) noexcept; - WriteResult write(ssize_t index, float v) noexcept; - WriteResult write(ssize_t index, double v) noexcept; - - WriteResult write(ssize_t index, int8_t v) noexcept; - WriteResult write(ssize_t index, int16_t v) noexcept; - WriteResult write(ssize_t index, int32_t v) noexcept; - WriteResult write(ssize_t index, int64_t v) noexcept; - WriteResult write(ssize_t index, uint8_t v) noexcept; - WriteResult write(ssize_t index, uint16_t v) noexcept; - WriteResult write(ssize_t index, uint32_t v) noexcept; - WriteResult write(ssize_t index, uint64_t v) noexcept; - - WriteResult write(ssize_t index, const std::string& v) noexcept; - WriteResult write(ssize_t index, folly::StringPiece v) noexcept; - WriteResult write(ssize_t index, const char* v) noexcept; - - WriteResult write(ssize_t index, const Date& v) noexcept; - WriteResult write(ssize_t index, const Time& v) noexcept; - WriteResult write(ssize_t index, const DateTime& v) noexcept; - WriteResult write(ssize_t index, const Duration& v) noexcept; - - WriteResult write(ssize_t index, const Geography& v) noexcept; + bool checkNullBit(ssize_t pos) const; + + WriteResult write(ssize_t index, bool v); + WriteResult write(ssize_t index, float v); + WriteResult write(ssize_t index, double v); + + WriteResult write(ssize_t index, int8_t v); + WriteResult write(ssize_t index, int16_t v); + WriteResult write(ssize_t index, int32_t v); + WriteResult write(ssize_t index, int64_t v); + WriteResult write(ssize_t index, uint8_t v); + WriteResult write(ssize_t index, uint16_t v); + WriteResult write(ssize_t index, uint32_t v); + WriteResult write(ssize_t index, uint64_t v); + + WriteResult write(ssize_t index, const std::string& v); + WriteResult write(ssize_t index, folly::StringPiece v); + WriteResult write(ssize_t index, const char* v); + + WriteResult write(ssize_t index, const Date& v); + WriteResult write(ssize_t index, const Time& v); + WriteResult write(ssize_t index, const DateTime& v); + WriteResult write(ssize_t index, const Duration& v); + + WriteResult write(ssize_t index, const Geography& v); }; } // namespace nebula diff --git a/src/common/base/ObjectPool.h b/src/common/base/ObjectPool.h index acd20d73986..ce0f420c365 100644 --- a/src/common/base/ObjectPool.h +++ b/src/common/base/ObjectPool.h @@ -38,9 +38,12 @@ class ObjectPool final : private boost::noncopyable, private cpp::NonMovable { template T *makeAndAdd(Args &&... args) { - lock_.lock(); - void *ptr = arena_.allocateAligned(sizeof(T)); - lock_.unlock(); + void *ptr; + { + // alloc happens here(may throw bad_alloc), use guard to guarantee unlock + SLGuard g(lock_); + ptr = arena_.allocateAligned(sizeof(T)); + } return add(new (ptr) T(std::forward(args)...)); } diff --git a/src/common/datatypes/DataSet.h b/src/common/datatypes/DataSet.h index b1230001ce8..600ef231ec8 100644 --- a/src/common/datatypes/DataSet.h +++ b/src/common/datatypes/DataSet.h @@ -26,7 +26,7 @@ struct DataSet { DataSet() = default; explicit DataSet(std::vector columns) : colNames(std::move(columns)) {} - DataSet(const DataSet& ds) noexcept { + DataSet(const DataSet& ds) { colNames = ds.colNames; rows = ds.rows; } @@ -34,7 +34,7 @@ struct DataSet { colNames = std::move(ds.colNames); rows = std::move(ds.rows); } - DataSet& operator=(const DataSet& ds) noexcept { + DataSet& operator=(const DataSet& ds) { if (&ds != this) { colNames = ds.colNames; rows = ds.rows; diff --git a/src/common/datatypes/Edge.cpp b/src/common/datatypes/Edge.cpp index c5d2f3464b4..c734db5aa4d 100644 --- a/src/common/datatypes/Edge.cpp +++ b/src/common/datatypes/Edge.cpp @@ -167,7 +167,7 @@ std::string Edge::id() const { namespace std { // Inject a customized hash function -std::size_t hash::operator()(const nebula::Edge& h) const noexcept { +std::size_t hash::operator()(const nebula::Edge& h) const { const auto& src = h.type > 0 ? h.src.toString() : h.dst.toString(); const auto& dst = h.type > 0 ? h.dst.toString() : h.src.toString(); auto type = h.type > 0 ? h.type : -h.type; diff --git a/src/common/datatypes/Edge.h b/src/common/datatypes/Edge.h index e873c62689a..dcdef8a58c7 100644 --- a/src/common/datatypes/Edge.h +++ b/src/common/datatypes/Edge.h @@ -100,7 +100,7 @@ namespace std { // Inject a customized hash function template <> struct hash { - std::size_t operator()(const nebula::Edge& h) const noexcept; + std::size_t operator()(const nebula::Edge& h) const; }; } // namespace std diff --git a/src/common/datatypes/Geography.cpp b/src/common/datatypes/Geography.cpp index 05885d294db..01eb56e6a7b 100644 --- a/src/common/datatypes/Geography.cpp +++ b/src/common/datatypes/Geography.cpp @@ -387,7 +387,7 @@ bool Geography::operator<(const Geography& rhs) const { namespace std { // Inject a customized hash function -std::size_t hash::operator()(const nebula::Geography& v) const noexcept { +std::size_t hash::operator()(const nebula::Geography& v) const { std::string wkb = v.asWKB(); return hash{}(wkb); } diff --git a/src/common/datatypes/Geography.h b/src/common/datatypes/Geography.h index 6d819750af4..2f93c157b3d 100644 --- a/src/common/datatypes/Geography.h +++ b/src/common/datatypes/Geography.h @@ -238,7 +238,7 @@ namespace std { // Inject a customized hash function template <> struct hash { - std::size_t operator()(const nebula::Geography& h) const noexcept; + std::size_t operator()(const nebula::Geography& h) const; }; } // namespace std diff --git a/src/common/datatypes/List.h b/src/common/datatypes/List.h index 28b55caf5cf..d2509d9b687 100644 --- a/src/common/datatypes/List.h +++ b/src/common/datatypes/List.h @@ -103,7 +103,7 @@ inline std::ostream& operator<<(std::ostream& os, const List& l) { namespace std { template <> struct hash { - std::size_t operator()(const nebula::List& h) const noexcept { + std::size_t operator()(const nebula::List& h) const { if (h.values.size() == 1) { return std::hash()(h.values[0]); } diff --git a/src/common/datatypes/Map.cpp b/src/common/datatypes/Map.cpp index 095d001cc4a..7cb1de2b80a 100644 --- a/src/common/datatypes/Map.cpp +++ b/src/common/datatypes/Map.cpp @@ -91,7 +91,7 @@ Map::Map(const folly::dynamic& obj) { } // namespace nebula namespace std { -std::size_t hash::operator()(const nebula::Map& m) const noexcept { +std::size_t hash::operator()(const nebula::Map& m) const { size_t seed = 0; for (auto& v : m.kvs) { seed ^= hash()(v.first) + 0x9e3779b9 + (seed << 6) + (seed >> 2); diff --git a/src/common/datatypes/Map.h b/src/common/datatypes/Map.h index 91134a48387..15a0dd9099e 100644 --- a/src/common/datatypes/Map.h +++ b/src/common/datatypes/Map.h @@ -88,7 +88,7 @@ inline std::ostream& operator<<(std::ostream& os, const Map& m) { namespace std { template <> struct hash { - std::size_t operator()(const nebula::Map& m) const noexcept; + std::size_t operator()(const nebula::Map& m) const; }; } // namespace std diff --git a/src/common/datatypes/Path.cpp b/src/common/datatypes/Path.cpp index b752239ed12..9ebb33be243 100644 --- a/src/common/datatypes/Path.cpp +++ b/src/common/datatypes/Path.cpp @@ -87,7 +87,7 @@ std::size_t hash::operator()(const nebula::Step& h) const noexcept return folly::hash::fnv64_buf(reinterpret_cast(&h.ranking), sizeof(h.ranking), hv); } -std::size_t hash::operator()(const nebula::Path& h) const noexcept { +std::size_t hash::operator()(const nebula::Path& h) const { size_t hv = hash()(h.src); for (auto& s : h.steps) { hv += (hv << 1) + (hv << 4) + (hv << 5) + (hv << 7) + (hv << 8) + (hv << 40); diff --git a/src/common/datatypes/Path.h b/src/common/datatypes/Path.h index 5dbedb3b9d0..cfc9c07c629 100644 --- a/src/common/datatypes/Path.h +++ b/src/common/datatypes/Path.h @@ -28,11 +28,7 @@ struct Step { name(std::move(s.name)), ranking(std::move(s.ranking)), props(std::move(s.props)) {} - Step(Vertex d, - EdgeType t, - std::string n, - EdgeRanking r, - std::unordered_map p) noexcept + Step(Vertex d, EdgeType t, std::string n, EdgeRanking r, std::unordered_map p) : dst(std::move(d)), type(t), name(std::move(n)), ranking(r), props(std::move(p)) {} void clear() { @@ -71,7 +67,7 @@ struct Step { return *this; } - Step& operator=(const Step& rhs) noexcept { + Step& operator=(const Step& rhs) { if (&rhs != this) { dst = rhs.dst; type = rhs.type; @@ -190,7 +186,7 @@ struct Path { return *this; } - Path& operator=(const Path& rhs) noexcept { + Path& operator=(const Path& rhs) { if (&rhs != this) { src = rhs.src; steps = rhs.steps; @@ -247,7 +243,7 @@ struct hash { template <> struct hash { - std::size_t operator()(const nebula::Path& h) const noexcept; + std::size_t operator()(const nebula::Path& h) const; }; } // namespace std diff --git a/src/common/datatypes/Set.cpp b/src/common/datatypes/Set.cpp index 103170546e4..92f116f2c10 100644 --- a/src/common/datatypes/Set.cpp +++ b/src/common/datatypes/Set.cpp @@ -44,7 +44,7 @@ folly::dynamic Set::getMetaData() const { } // namespace nebula namespace std { -std::size_t hash::operator()(const nebula::Set& s) const noexcept { +std::size_t hash::operator()(const nebula::Set& s) const { size_t seed = 0; for (auto& v : s.values) { seed ^= hash()(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); diff --git a/src/common/datatypes/Set.h b/src/common/datatypes/Set.h index 107cf3aee5f..5e3e30870b8 100644 --- a/src/common/datatypes/Set.h +++ b/src/common/datatypes/Set.h @@ -72,7 +72,7 @@ inline std::ostream& operator<<(std::ostream& os, const Set& s) { namespace std { template <> struct hash { - std::size_t operator()(const nebula::Set& s) const noexcept; + std::size_t operator()(const nebula::Set& s) const; }; } // namespace std diff --git a/src/common/datatypes/Value.cpp b/src/common/datatypes/Value.cpp index f17d9a4642b..12d2b143ada 100644 --- a/src/common/datatypes/Value.cpp +++ b/src/common/datatypes/Value.cpp @@ -1185,265 +1185,265 @@ Value& Value::operator=(const Value& rhs) { } void Value::setN(const NullType& v) { - type_ = Type::NULLVALUE; new (std::addressof(value_.nVal)) NullType(v); + type_ = Type::NULLVALUE; } void Value::setN(NullType&& v) { - type_ = Type::NULLVALUE; new (std::addressof(value_.nVal)) NullType(std::move(v)); + type_ = Type::NULLVALUE; } void Value::setB(const bool& v) { - type_ = Type::BOOL; new (std::addressof(value_.bVal)) bool(v); // NOLINT + type_ = Type::BOOL; } void Value::setB(bool&& v) { - type_ = Type::BOOL; new (std::addressof(value_.bVal)) bool(std::move(v)); // NOLINT + type_ = Type::BOOL; } void Value::setI(const int64_t& v) { - type_ = Type::INT; new (std::addressof(value_.iVal)) int64_t(v); // NOLINT + type_ = Type::INT; } void Value::setI(int64_t&& v) { - type_ = Type::INT; new (std::addressof(value_.iVal)) int64_t(std::move(v)); // NOLINT + type_ = Type::INT; } void Value::setF(const double& v) { - type_ = Type::FLOAT; new (std::addressof(value_.fVal)) double(v); // NOLINT + type_ = Type::FLOAT; } void Value::setF(double&& v) { - type_ = Type::FLOAT; new (std::addressof(value_.fVal)) double(std::move(v)); // NOLINT + type_ = Type::FLOAT; } void Value::setS(std::unique_ptr v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(std::move(v)); + type_ = Type::STRING; } void Value::setS(const std::string& v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(v)); + type_ = Type::STRING; } void Value::setS(std::string&& v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(std::move(v))); + type_ = Type::STRING; } void Value::setS(const char* v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(v)); + type_ = Type::STRING; } void Value::setD(const Date& v) { - type_ = Type::DATE; new (std::addressof(value_.dVal)) Date(v); + type_ = Type::DATE; } void Value::setD(Date&& v) { - type_ = Type::DATE; new (std::addressof(value_.dVal)) Date(std::move(v)); + type_ = Type::DATE; } void Value::setT(const Time& v) { - type_ = Type::TIME; new (std::addressof(value_.tVal)) Time(v); + type_ = Type::TIME; } void Value::setT(Time&& v) { - type_ = Type::TIME; new (std::addressof(value_.tVal)) Time(std::move(v)); + type_ = Type::TIME; } void Value::setDT(const DateTime& v) { - type_ = Type::DATETIME; new (std::addressof(value_.dtVal)) DateTime(v); + type_ = Type::DATETIME; } void Value::setDT(DateTime&& v) { - type_ = Type::DATETIME; new (std::addressof(value_.dtVal)) DateTime(std::move(v)); + type_ = Type::DATETIME; } void Value::setV(Vertex* v) { - type_ = Type::VERTEX; value_.vVal = v; value_.vVal->ref(); + type_ = Type::VERTEX; } void Value::setV(const Vertex& v) { - type_ = Type::VERTEX; new (std::addressof(value_.vVal)) Vertex*(new Vertex(v)); + type_ = Type::VERTEX; } void Value::setV(Vertex&& v) { - type_ = Type::VERTEX; new (std::addressof(value_.vVal)) Vertex*(new Vertex(std::move(v))); + type_ = Type::VERTEX; } void Value::setE(Edge* v) { - type_ = Type::EDGE; value_.eVal = v; value_.eVal->ref(); + type_ = Type::EDGE; } void Value::setE(const Edge& v) { - type_ = Type::EDGE; new (std::addressof(value_.eVal)) Edge*(new Edge(v)); + type_ = Type::EDGE; } void Value::setE(Edge&& v) { - type_ = Type::EDGE; new (std::addressof(value_.eVal)) Edge*(new Edge(std::move(v))); + type_ = Type::EDGE; } void Value::setP(const std::unique_ptr& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(new Path(*v)); + type_ = Type::PATH; } void Value::setP(std::unique_ptr&& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(std::move(v)); + type_ = Type::PATH; } void Value::setP(const Path& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(new Path(v)); + type_ = Type::PATH; } void Value::setP(Path&& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(new Path(std::move(v))); + type_ = Type::PATH; } void Value::setL(const std::unique_ptr& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(new List(*v)); + type_ = Type::LIST; } void Value::setL(std::unique_ptr&& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(std::move(v)); + type_ = Type::LIST; } void Value::setL(const List& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(new List(v)); + type_ = Type::LIST; } void Value::setL(List&& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(new List(std::move(v))); + type_ = Type::LIST; } void Value::setM(const std::unique_ptr& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(new Map(*v)); + type_ = Type::MAP; } void Value::setM(std::unique_ptr&& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(std::move(v)); + type_ = Type::MAP; } void Value::setM(const Map& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(new Map(v)); + type_ = Type::MAP; } void Value::setM(Map&& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(new Map(std::move(v))); + type_ = Type::MAP; } void Value::setU(const std::unique_ptr& v) { - type_ = Type::SET; new (std::addressof(value_.uVal)) std::unique_ptr(new Set(*v)); + type_ = Type::SET; } void Value::setU(std::unique_ptr&& v) { - type_ = Type::SET; new (std::addressof(value_.uVal)) std::unique_ptr(std::move(v)); + type_ = Type::SET; } void Value::setU(const Set& v) { - type_ = Type::SET; new (std::addressof(value_.uVal)) std::unique_ptr(new Set(v)); + type_ = Type::SET; } void Value::setU(Set&& v) { - type_ = Type::SET; new (std::addressof(value_.vVal)) std::unique_ptr(new Set(std::move(v))); + type_ = Type::SET; } void Value::setG(const std::unique_ptr& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(*v)); + type_ = Type::DATASET; } void Value::setG(std::unique_ptr&& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(std::move(v)); + type_ = Type::DATASET; } void Value::setG(const DataSet& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(v)); + type_ = Type::DATASET; } void Value::setG(DataSet&& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(std::move(v))); + type_ = Type::DATASET; } void Value::setGG(const std::unique_ptr& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(*v)); + type_ = Type::GEOGRAPHY; } void Value::setGG(std::unique_ptr&& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(std::move(v)); + type_ = Type::GEOGRAPHY; } void Value::setGG(const Geography& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(v)); + type_ = Type::GEOGRAPHY; } void Value::setGG(Geography&& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(std::move(v))); + type_ = Type::GEOGRAPHY; } void Value::setDU(const std::unique_ptr& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(*v)); + type_ = Type::DURATION; } void Value::setDU(std::unique_ptr&& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(std::move(v)); + type_ = Type::DURATION; } void Value::setDU(const Duration& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(v)); + type_ = Type::DURATION; } void Value::setDU(Duration&& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(std::move(v))); + type_ = Type::DURATION; } // Convert Nebula::Value to a value compatible with Json standard diff --git a/src/common/datatypes/Value.h b/src/common/datatypes/Value.h index 7047ccd0f53..6d693533995 100644 --- a/src/common/datatypes/Value.h +++ b/src/common/datatypes/Value.h @@ -557,7 +557,7 @@ namespace std { // Inject a customized hash function template <> struct hash { - std::size_t operator()(const nebula::Value& h) const noexcept { + std::size_t operator()(const nebula::Value& h) const { if (h.isInt()) { return h.getInt(); } else if (h.isStr()) { @@ -569,14 +569,14 @@ struct hash { template <> struct hash { - std::size_t operator()(const nebula::Value* h) const noexcept { + std::size_t operator()(const nebula::Value* h) const { return h == nullptr ? 0 : hash()(*h); } }; template <> struct hash { - std::size_t operator()(const nebula::Value* h) const noexcept { + std::size_t operator()(const nebula::Value* h) const { return h == nullptr ? 0 : hash()(*h); } }; diff --git a/src/common/datatypes/ValueOps-inl.h b/src/common/datatypes/ValueOps-inl.h index c58cb0a5261..08ff4205f2c 100644 --- a/src/common/datatypes/ValueOps-inl.h +++ b/src/common/datatypes/ValueOps-inl.h @@ -369,10 +369,10 @@ void Cpp2Ops::read(Protocol* proto, nebula::Value* obj) { } case 9: { if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { - auto* ptr = new nebula::Vertex(); - Cpp2Ops::read(proto, ptr); - obj->setVertex(ptr); + auto ptr = std::make_unique(); ptr->unref(); + Cpp2Ops::read(proto, ptr.get()); + obj->setVertex(ptr.release()); } else { proto->skip(readState.fieldType); } @@ -380,10 +380,10 @@ void Cpp2Ops::read(Protocol* proto, nebula::Value* obj) { } case 10: { if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { - auto* ptr = new nebula::Edge(); - Cpp2Ops::read(proto, ptr); - obj->setEdge(ptr); + auto ptr = std::make_unique(); ptr->unref(); + Cpp2Ops::read(proto, ptr.get()); + obj->setEdge(ptr.release()); } else { proto->skip(readState.fieldType); } diff --git a/src/common/datatypes/Vertex.cpp b/src/common/datatypes/Vertex.cpp index 291e6cfd6f1..41eef5208a4 100644 --- a/src/common/datatypes/Vertex.cpp +++ b/src/common/datatypes/Vertex.cpp @@ -133,7 +133,7 @@ std::size_t hash::operator()(const nebula::Tag& h) const noexcept { return folly::hash::fnv64(h.name); } -std::size_t hash::operator()(const nebula::Vertex& h) const noexcept { +std::size_t hash::operator()(const nebula::Vertex& h) const { size_t hv = folly::hash::fnv64(h.vid.toString()); for (auto& t : h.tags) { hv += (hv << 1) + (hv << 4) + (hv << 5) + (hv << 7) + (hv << 8) + (hv << 40); diff --git a/src/common/datatypes/Vertex.h b/src/common/datatypes/Vertex.h index 29ef6f83753..539d256f28a 100644 --- a/src/common/datatypes/Vertex.h +++ b/src/common/datatypes/Vertex.h @@ -130,7 +130,7 @@ struct hash { template <> struct hash { - std::size_t operator()(const nebula::Vertex& h) const noexcept; + std::size_t operator()(const nebula::Vertex& h) const; }; } // namespace std diff --git a/src/common/expression/Expression.cpp b/src/common/expression/Expression.cpp index ec36dab0b67..478bdbe3c71 100644 --- a/src/common/expression/Expression.cpp +++ b/src/common/expression/Expression.cpp @@ -50,12 +50,12 @@ std::string Expression::Encoder::moveStr() { return std::move(buf_); } -Expression::Encoder& Expression::Encoder::operator<<(Kind kind) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(Kind kind) { buf_.append(reinterpret_cast(&kind), sizeof(uint8_t)); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(const std::string& str) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(const std::string& str) { size_t sz = str.size(); buf_.append(reinterpret_cast(&sz), sizeof(size_t)); if (sz > 0) { @@ -64,22 +64,22 @@ Expression::Encoder& Expression::Encoder::operator<<(const std::string& str) noe return *this; } -Expression::Encoder& Expression::Encoder::operator<<(const Value& val) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(const Value& val) { serializer::serialize(val, &buf_); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(size_t size) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(size_t size) { buf_.append(reinterpret_cast(&size), sizeof(size_t)); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(Value::Type vType) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(Value::Type vType) { buf_.append(reinterpret_cast(&vType), sizeof(Value::Type)); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(const Expression& exp) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(const Expression& exp) { exp.writeTo(*this); return *this; } @@ -100,7 +100,7 @@ std::string Expression::Decoder::getHexStr() const { return toHexStr(encoded_); } -Expression::Kind Expression::Decoder::readKind() noexcept { +Expression::Kind Expression::Decoder::readKind() { CHECK_LE(ptr_ + sizeof(uint8_t), encoded_.end()); Expression::Kind kind; @@ -110,7 +110,7 @@ Expression::Kind Expression::Decoder::readKind() noexcept { return kind; } -std::string Expression::Decoder::readStr() noexcept { +std::string Expression::Decoder::readStr() { CHECK_LE(ptr_ + sizeof(size_t), encoded_.end()); size_t sz = 0; @@ -127,7 +127,7 @@ std::string Expression::Decoder::readStr() noexcept { return std::string(ptr, sz); } -Value Expression::Decoder::readValue() noexcept { +Value Expression::Decoder::readValue() { Value val; size_t len = serializer::deserialize(folly::StringPiece(ptr_, encoded_.end()), val); ptr_ += len; @@ -148,7 +148,7 @@ Value::Type Expression::Decoder::readValueType() noexcept { return type; } -Expression* Expression::Decoder::readExpression(ObjectPool* pool) noexcept { +Expression* Expression::Decoder::readExpression(ObjectPool* pool) { return Expression::decode(pool, *this); } diff --git a/src/common/expression/Expression.h b/src/common/expression/Expression.h index 0b21287a5fd..a23340589fe 100644 --- a/src/common/expression/Expression.h +++ b/src/common/expression/Expression.h @@ -177,12 +177,12 @@ class Expression { explicit Encoder(size_t bufSizeHint = 2048); std::string moveStr(); - Encoder& operator<<(Kind kind) noexcept; - Encoder& operator<<(const std::string& str) noexcept; - Encoder& operator<<(const Value& val) noexcept; - Encoder& operator<<(size_t size) noexcept; - Encoder& operator<<(Value::Type vType) noexcept; - Encoder& operator<<(const Expression& exp) noexcept; + Encoder& operator<<(Kind kind); + Encoder& operator<<(const std::string& str); + Encoder& operator<<(const Value& val); + Encoder& operator<<(size_t size); + Encoder& operator<<(Value::Type vType); + Encoder& operator<<(const Expression& exp); private: std::string buf_; @@ -194,12 +194,12 @@ class Expression { bool finished() const; - Kind readKind() noexcept; - std::string readStr() noexcept; - Value readValue() noexcept; + Kind readKind(); + std::string readStr(); + Value readValue(); size_t readSize() noexcept; Value::Type readValueType() noexcept; - Expression* readExpression(ObjectPool* pool) noexcept; + Expression* readExpression(ObjectPool* pool); // Convert the unprocessed part into the hex string std::string getHexStr() const; diff --git a/src/common/geo/GeoIndex.cpp b/src/common/geo/GeoIndex.cpp index 0995afc99eb..b041557216e 100644 --- a/src/common/geo/GeoIndex.cpp +++ b/src/common/geo/GeoIndex.cpp @@ -57,7 +57,7 @@ nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() const { return hint; } -std::vector GeoIndex::indexCells(const Geography& g) const noexcept { +std::vector GeoIndex::indexCells(const Geography& g) const { auto r = g.asS2(); if (UNLIKELY(!r)) { return {}; @@ -72,7 +72,7 @@ std::vector GeoIndex::indexCells(const Geography& g) const noexcept { return cellIds; } -std::vector GeoIndex::intersects(const Geography& g) const noexcept { +std::vector GeoIndex::intersects(const Geography& g) const { auto r = g.asS2(); if (UNLIKELY(!r)) { return {}; @@ -82,16 +82,16 @@ std::vector GeoIndex::intersects(const Geography& g) const noexcept { } // covers degenerates to intersects currently -std::vector GeoIndex::covers(const Geography& g) const noexcept { +std::vector GeoIndex::covers(const Geography& g) const { return intersects(g); } // coveredBy degenerates to intersects currently -std::vector GeoIndex::coveredBy(const Geography& g) const noexcept { +std::vector GeoIndex::coveredBy(const Geography& g) const { return intersects(g); } -std::vector GeoIndex::dWithin(const Geography& g, double distance) const noexcept { +std::vector GeoIndex::dWithin(const Geography& g, double distance) const { auto r = g.asS2(); if (UNLIKELY(!r)) { return {}; @@ -124,7 +124,7 @@ std::vector GeoIndex::dWithin(const Geography& g, double distance) co } } -std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) const noexcept { +std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) const { auto cells = coveringCells(r, isPoint); std::vector scanRanges; for (const S2CellId& cellId : cells) { @@ -147,7 +147,7 @@ std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) con return scanRanges; } -std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) const noexcept { +std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) const { // Currently we don't apply region coverer params to point, because it's useless. // Point always use level 30. if (isPoint) { @@ -166,7 +166,7 @@ std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) c return covering; } -std::vector GeoIndex::ancestorCells(const std::vector& cells) const noexcept { +std::vector GeoIndex::ancestorCells(const std::vector& cells) const { // DCHECK(rc.IsCanonical(cells)); std::vector ancestors; std::unordered_set seen; diff --git a/src/common/geo/GeoIndex.h b/src/common/geo/GeoIndex.h index 3b1c7e618f4..cdd5032ed97 100644 --- a/src/common/geo/GeoIndex.h +++ b/src/common/geo/GeoIndex.h @@ -64,24 +64,24 @@ class GeoIndex { : rcParams_(params), pointsOnly_(pointsOnly) {} // Build geo index for the geography g - std::vector indexCells(const Geography& g) const noexcept; + std::vector indexCells(const Geography& g) const; // Query the geo index // ST_Intersects(g, x), x is the indexed geography column - std::vector intersects(const Geography& g) const noexcept; + std::vector intersects(const Geography& g) const; // ST_Covers(g, x), x is the indexed geography column - std::vector covers(const Geography& g) const noexcept; + std::vector covers(const Geography& g) const; // ST_CoveredBy(g, x), x is the indexed geography column - std::vector coveredBy(const Geography& g) const noexcept; + std::vector coveredBy(const Geography& g) const; // ST_Distance(g, x, distance), x is the indexed geography column - std::vector dWithin(const Geography& g, double distance) const noexcept; + std::vector dWithin(const Geography& g, double distance) const; private: - std::vector intersects(const S2Region& r, bool isPoint = false) const noexcept; + std::vector intersects(const S2Region& r, bool isPoint = false) const; - std::vector coveringCells(const S2Region& r, bool isPoint = false) const noexcept; + std::vector coveringCells(const S2Region& r, bool isPoint = false) const; - std::vector ancestorCells(const std::vector& cells) const noexcept; + std::vector ancestorCells(const std::vector& cells) const; private: RegionCoverParams rcParams_; diff --git a/src/common/memory/Memory.h b/src/common/memory/Memory.h index 418685900bd..c0a43ba2c17 100644 --- a/src/common/memory/Memory.h +++ b/src/common/memory/Memory.h @@ -105,11 +105,21 @@ inline ALWAYS_INLINE size_t getActualAllocationSize(size_t size, std::align_val_ } inline ALWAYS_INLINE void trackMemory(std::size_t size) { + std::size_t actual_size = getActualAllocationSize(size); + MemoryTracker::alloc(actual_size); +} + +inline ALWAYS_INLINE void trackMemoryNoThrow(std::size_t size) { std::size_t actual_size = getActualAllocationSize(size); MemoryTracker::allocNoThrow(actual_size); } inline ALWAYS_INLINE void trackMemory(std::size_t size, std::align_val_t align) { + std::size_t actual_size = getActualAllocationSize(size, align); + MemoryTracker::alloc(actual_size); +} + +inline ALWAYS_INLINE void trackMemoryNoThrow(std::size_t size, std::align_val_t align) { std::size_t actual_size = getActualAllocationSize(size, align); MemoryTracker::allocNoThrow(actual_size); } diff --git a/src/common/memory/MemoryTracker.cpp b/src/common/memory/MemoryTracker.cpp index a22a0aabf0d..cae9c756b72 100644 --- a/src/common/memory/MemoryTracker.cpp +++ b/src/common/memory/MemoryTracker.cpp @@ -41,8 +41,8 @@ bool MemoryTracker::isOn() { return MemoryStats::instance().throwOnMemoryExceeded(); } -void MemoryTracker::allocImpl(int64_t size, bool) { - MemoryStats::instance().alloc(size); +void MemoryTracker::allocImpl(int64_t size, bool throw_if_memory_exceeded) { + MemoryStats::instance().alloc(size, throw_if_memory_exceeded); } } // namespace memory diff --git a/src/common/memory/MemoryTracker.h b/src/common/memory/MemoryTracker.h index f42d5223ad9..a1c4a646c6a 100644 --- a/src/common/memory/MemoryTracker.h +++ b/src/common/memory/MemoryTracker.h @@ -63,7 +63,7 @@ class MemoryStats { } /// Inform size of memory allocation - inline ALWAYS_INLINE void alloc(int64_t size) { + inline ALWAYS_INLINE void alloc(int64_t size, bool throw_if_memory_exceeded) { int64_t willBe = threadMemoryStats_.reserved - size; if (UNLIKELY(willBe < 0)) { @@ -74,7 +74,7 @@ class MemoryStats { } // allocGlobal() may end with bad_alloc, only invoke allocGlobal() once (ALL_OR_NOTHING // semantic) - allocGlobal(getFromGlobal); + allocGlobal(getFromGlobal, throw_if_memory_exceeded); willBe += getFromGlobal; } // Only update after successful allocations, failed allocations should not be taken into @@ -103,12 +103,19 @@ class MemoryStats { /// Set limit (maximum usable bytes) of memory void setLimit(int64_t limit) { if (this->limit_ != limit) { - LOG(INFO) << fmt::format( + DLOG(INFO) << fmt::format( "MemoryTracker update limit {} -> {}", ReadableSize(this->limit_), ReadableSize(limit)); this->limit_ = limit; } } + /// update limit (maximum usable bytes) of memory + /// limit will be set to "used memory + available" + void updateLimit(int64_t available) { + int64_t newLimit = used_ + available; + setLimit(newLimit); + } + /// Get limit (maximum usable bytes) of memory int64_t getLimit() { return this->limit_; @@ -143,12 +150,17 @@ class MemoryStats { return threadMemoryStats_.throwOnMemoryExceeded; } + static bool setThrowOnMemoryExceeded(bool value) { + return threadMemoryStats_.throwOnMemoryExceeded = value; + } + private: - inline ALWAYS_INLINE void allocGlobal(int64_t size) { + inline ALWAYS_INLINE void allocGlobal(int64_t size, bool throw_if_memory_exceeded) { int64_t willBe = size + used_.fetch_add(size, std::memory_order_relaxed); - if (threadMemoryStats_.throwOnMemoryExceeded && willBe > limit_) { + if (threadMemoryStats_.throwOnMemoryExceeded && throw_if_memory_exceeded && willBe > limit_) { // revert used_.fetch_sub(size, std::memory_order_relaxed); + threadMemoryStats_.throwOnMemoryExceeded = false; throw std::bad_alloc(); } } @@ -165,13 +177,27 @@ class MemoryStats { // A guard to only enable memory check (throw when memory exceed) during its lifetime. struct MemoryCheckGuard { + bool previous; MemoryCheckGuard() { + previous = MemoryStats::throwOnMemoryExceeded(); MemoryStats::turnOnThrow(); } ~MemoryCheckGuard() { + MemoryStats::setThrowOnMemoryExceeded(previous); + } +}; + +struct MemoryCheckOffGuard { + bool previous; + MemoryCheckOffGuard() { + previous = MemoryStats::throwOnMemoryExceeded(); MemoryStats::turnOffThrow(); } + + ~MemoryCheckOffGuard() { + MemoryStats::setThrowOnMemoryExceeded(previous); + } }; // A global static memory tracker enable tracking every memory allocation and deallocation. diff --git a/src/common/memory/MemoryUtils.cpp b/src/common/memory/MemoryUtils.cpp index f8950fd3d21..52d832af937 100644 --- a/src/common/memory/MemoryUtils.cpp +++ b/src/common/memory/MemoryUtils.cpp @@ -6,6 +6,9 @@ #include "common/memory/MemoryUtils.h" #include + +#include + #if ENABLE_JEMALLOC #include @@ -55,7 +58,11 @@ DEFINE_double(memory_tracker_untracked_reserved_memory_mb, "memory (direct call malloc/free)"); DEFINE_double(memory_tracker_limit_ratio, 0.8, - "memory tacker usable memory ratio to (total_available - untracked_reserved_memory)"); + "memory tacker usable memory ratio to (total - untracked_reserved_memory)"); + +DEFINE_double(memory_tracker_available_ratio, + 0.8, + "memory tacker available memory ratio to (available - untracked_reserved_memory)"); using nebula::fs::FileUtils; @@ -70,6 +77,8 @@ static const std::regex reTotalCache(R"(^total_(cache|inactive_file)\s+(\d+)$)") std::atomic_bool MemoryUtils::kHitMemoryHighWatermark{false}; int64_t MemoryUtils::kLastPurge_{0}; int64_t MemoryUtils::kLastPrintMemoryTrackerStats_{0}; +int64_t MemoryUtils::kCurrentTotal_{0}; +double MemoryUtils::kCurrentLimitRatio_{0.0}; StatusOr MemoryUtils::hitsHighWatermark() { if (FLAGS_system_memory_high_watermark_ratio >= 1.0) { @@ -118,18 +127,77 @@ StatusOr MemoryUtils::hitsHighWatermark() { } } - // MemoryStats depends on jemalloc -#ifdef ENABLE_JEMALLOC -#ifndef ENABLE_ASAN - // set MemoryStats limit (MemoryTracker track-able memory) - int64_t trackable = total - FLAGS_memory_tracker_untracked_reserved_memory_mb * MiB; - if (trackable > 0) { - MemoryStats::instance().setLimit(trackable * FLAGS_memory_tracker_limit_ratio); + handleMemoryTracker(total, available); + + auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio; + LOG_IF_EVERY_N(WARNING, hits, 100) + << "Memory usage has hit the high watermark of system, available: " << available + << " vs. total: " << total << " in bytes."; + return hits; +} + +StatusOr MemoryUtils::readSysContents(const std::string& path) { + std::ifstream ifs(path); + if (!ifs) { + return Status::Error("Could not open the file: %s", path.c_str()); + } + uint64_t value = 0; + ifs >> value; + return value; +} + +void MemoryUtils::handleMemoryTracker(int64_t total, int64_t available) { +#ifdef ENABLE_MEMORY_TRACKER + double limitRatio = FLAGS_memory_tracker_limit_ratio; + double availableRatio = FLAGS_memory_tracker_available_ratio; + + double untrackedMb = FLAGS_memory_tracker_untracked_reserved_memory_mb; + + // update MemoryTracker limit memory by Flags + if (limitRatio > 0.0 && limitRatio <= 1.0) { + /** + * (<= 1.0): Normal limit is set to ratio of trackable memory; + * limit = limitRatio * (total - untrackedMb) + */ + if (kCurrentTotal_ != total || kCurrentLimitRatio_ != limitRatio) { + double trackable = total - (untrackedMb * MiB); + if (trackable > 0) { + MemoryStats::instance().setLimit(trackable * limitRatio); + } else { + LOG(ERROR) << "Total memory less than untracked " << untrackedMb << " Mib," + << " MemoryTracker limit is not set properly, " + << " Current memory stats:" << MemoryStats::instance().toString(); + } + LOG(INFO) << "MemoryTracker set static ratio: " << limitRatio; + } + } else if (std::fabs(limitRatio - 2.0) < std::numeric_limits::epsilon()) { + /** + * (== 2.0): Special value for dynamic self adaptive; + * limit = current_used_memory + (available - untracked) * availableRatio + */ + double trackableAvailable = available - (untrackedMb * MiB); + if (trackableAvailable > 0) { + MemoryStats::instance().updateLimit(trackableAvailable * availableRatio); + } else { + MemoryStats::instance().updateLimit(0); + } + DLOG(INFO) << "MemoryTracker set to dynamic self adaptive"; + } else if (std::fabs(limitRatio - 3.0) < std::numeric_limits::epsilon()) { + /** + * (== 3.0): Special value for disable memory_tracker; + * limit is set to max + */ + if (kCurrentLimitRatio_ != limitRatio) { + MemoryStats::instance().setLimit(std::numeric_limits::max()); + LOG(INFO) << "MemoryTracker disabled"; + } } else { - // Do not set limit, keep previous set limit or default limit - LOG(ERROR) << "Total available memory less than " - << FLAGS_memory_tracker_untracked_reserved_memory_mb << " Mib"; + if (kCurrentTotal_ != limitRatio) { + LOG(ERROR) << "Invalid memory_tracker_limit_ratio: " << limitRatio; + } } + kCurrentTotal_ = total; + kCurrentLimitRatio_ = limitRatio; // purge if enabled if (FLAGS_memory_purge_enabled) { @@ -142,24 +210,26 @@ StatusOr MemoryUtils::hitsHighWatermark() { } } - // print system & application level memory stats - // sys: read from system environment, varies depends on environment: - // container: controlled by cgroup, - // used: read from memory.current in cgroup path - // total: read from memory.max in cgroup path - // physical machine: judge by system level memory consumption - // used: current used memory of the system - // total: all physical memory installed - // usr: record by current process's MemoryStats - // used: bytes allocated by new operator - // total: sys_total * FLAGS_system_memory_high_watermark_ratio + /** + * print system & application level memory stats + * sys: read from system environment, varies depends on environment: + * container: controlled by cgroup, + * used: read from memory.current in cgroup path + * total: read from memory.max in cgroup path + * physical machine: judge by system level memory consumption + * used: current used memory of the system + * total: all physical memory installed + * usr: record by current process's MemoryStats + * used: bytes allocated by new operator + * total: sys_total * FLAGS_system_memory_high_watermark_ratio + */ int64_t now = time::WallClock::fastNowInMilliSec(); if (FLAGS_memory_tracker_detail_log) { if (now - kLastPrintMemoryTrackerStats_ >= FLAGS_memory_tracker_detail_log_interval_ms) { LOG(INFO) << fmt::format("sys:{}/{} {:.2f}%", - ReadableSize(static_cast(total - available)), + ReadableSize(total - available), ReadableSize(total), - (1 - available / total) * 100) + (1 - available / static_cast(total)) * 100) << fmt::format(" usr:{}/{} {:.2f}%", ReadableSize(MemoryStats::instance().used()), ReadableSize(MemoryStats::instance().getLimit()), @@ -167,25 +237,12 @@ StatusOr MemoryUtils::hitsHighWatermark() { kLastPrintMemoryTrackerStats_ = now; } } +#else + LOG_FIRST_N(WARNING, 1) << "WARNING: MemoryTracker was disabled at compile time"; + UNUSED(total); + UNUSED(available); #endif -#endif - - auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio; - LOG_IF_EVERY_N(WARNING, hits, 100) - << "Memory usage has hit the high watermark of system, available: " << available - << " vs. total: " << total << " in bytes."; - return hits; -} - -StatusOr MemoryUtils::readSysContents(const std::string& path) { - std::ifstream ifs(path); - if (!ifs) { - return Status::Error("Could not open the file: %s", path.c_str()); - } - uint64_t value = 0; - ifs >> value; - return value; -} +} // namespace memory } // namespace memory } // namespace nebula diff --git a/src/common/memory/MemoryUtils.h b/src/common/memory/MemoryUtils.h index 90a471bb2c3..1fdf09d1c9b 100644 --- a/src/common/memory/MemoryUtils.h +++ b/src/common/memory/MemoryUtils.h @@ -33,9 +33,15 @@ class MemoryUtils final { static StatusOr readSysContents(const std::string &path); + // Handle memory tracker related logic by inform system's total memory and current available + // memory in bytes + static void handleMemoryTracker(int64_t total, int64_t available); + private: static int64_t kLastPurge_; static int64_t kLastPrintMemoryTrackerStats_; + static int64_t kCurrentTotal_; + static double kCurrentLimitRatio_; }; } // namespace memory diff --git a/src/common/memory/NewDelete.cpp b/src/common/memory/NewDelete.cpp index 5bc67076553..930cfe1f8e0 100644 --- a/src/common/memory/NewDelete.cpp +++ b/src/common/memory/NewDelete.cpp @@ -7,18 +7,12 @@ #include "common/memory/Memory.h" /// Replace default new/delete with memory tracking versions. -/// -/// Two condition need check before MemoryTracker is on +/// ENABLE_MEMORY_TRACKER is defined by cmake only following two condition satisfied: /// 1. jemalloc is used /// MemoryTracker need jemalloc API to get accurate size of alloc/free memory. /// 2. address_sanitizer is off /// sanitizer has already override the new/delete operator, /// only override new/delete operator only when address_sanitizer is off -#ifdef ENABLE_JEMALLOC -#ifndef ENABLE_ASAN -#define ENABLE_MEMORY_TRACKER -#endif -#endif #ifdef ENABLE_MEMORY_TRACKER /// new @@ -43,22 +37,22 @@ void *operator new[](std::size_t size, std::align_val_t align) { } void *operator new(std::size_t size, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size); + nebula::memory::trackMemoryNoThrow(size); return nebula::memory::newNoException(size); } void *operator new[](std::size_t size, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size); + nebula::memory::trackMemoryNoThrow(size); return nebula::memory::newNoException(size); } void *operator new(std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size, align); + nebula::memory::trackMemoryNoThrow(size, align); return nebula::memory::newNoException(size, align); } void *operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size, align); + nebula::memory::trackMemoryNoThrow(size, align); return nebula::memory::newNoException(size, align); } @@ -103,4 +97,6 @@ void operator delete[](void *ptr, std::size_t size, std::align_val_t align) noex nebula::memory::deleteSized(ptr, size, align); } +#else +#pragma message("WARNING: MemoryTracker was disabled at compile time") #endif diff --git a/src/graph/context/ExecutionContext.cpp b/src/graph/context/ExecutionContext.cpp index 1f1baae0429..bca879a329b 100644 --- a/src/graph/context/ExecutionContext.cpp +++ b/src/graph/context/ExecutionContext.cpp @@ -21,12 +21,16 @@ void ExecutionContext::setValue(const std::string& name, Value&& val) { } void ExecutionContext::setResult(const std::string& name, Result&& result) { + folly::RWSpinLock::WriteHolder holder(lock_); auto& hist = valueMap_[name]; hist.emplace_back(std::move(result)); } void ExecutionContext::dropResult(const std::string& name) { - DCHECK_EQ(valueMap_.count(name), 1); + folly::RWSpinLock::WriteHolder holder(lock_); + if (valueMap_.count(name) == 0) { + return; + } auto& val = valueMap_[name]; if (FLAGS_enable_async_gc) { GC::instance().clear(std::move(val)); @@ -36,6 +40,7 @@ void ExecutionContext::dropResult(const std::string& name) { } size_t ExecutionContext::numVersions(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); auto it = valueMap_.find(name); CHECK(it != valueMap_.end()); return it->second.size(); @@ -43,6 +48,7 @@ size_t ExecutionContext::numVersions(const std::string& name) const { // Only keep the last several versions of the Value void ExecutionContext::truncHistory(const std::string& name, size_t numVersionsToKeep) { + folly::RWSpinLock::WriteHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end()) { if (it->second.size() <= numVersionsToKeep) { @@ -59,6 +65,7 @@ const Value& ExecutionContext::getValue(const std::string& name) const { } Value ExecutionContext::moveValue(const std::string& name) { + folly::RWSpinLock::WriteHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end() && !it->second.empty()) { return it->second.back().moveValue(); @@ -68,6 +75,7 @@ Value ExecutionContext::moveValue(const std::string& name) { } const Result& ExecutionContext::getResult(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end() && !it->second.empty()) { return it->second.back(); @@ -79,6 +87,7 @@ const Result& ExecutionContext::getResult(const std::string& name) const { void ExecutionContext::setVersionedResult(const std::string& name, Result&& result, int64_t version) { + folly::RWSpinLock::WriteHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end()) { auto& hist = it->second; @@ -101,6 +110,7 @@ const Result& ExecutionContext::getVersionedResult(const std::string& name, int6 } const std::vector& ExecutionContext::getHistory(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end()) { return it->second; diff --git a/src/graph/context/ExecutionContext.h b/src/graph/context/ExecutionContext.h index d34acddc6a9..64cc02dfd5e 100644 --- a/src/graph/context/ExecutionContext.h +++ b/src/graph/context/ExecutionContext.h @@ -41,6 +41,7 @@ class ExecutionContext { virtual ~ExecutionContext() = default; void initVar(const std::string& name) { + folly::RWSpinLock::WriteHolder holder(lock_); valueMap_[name]; } @@ -69,6 +70,7 @@ class ExecutionContext { void truncHistory(const std::string& name, size_t numVersionsToKeep); bool exist(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); return valueMap_.find(name) != valueMap_.end(); } @@ -77,6 +79,7 @@ class ExecutionContext { Value moveValue(const std::string& name); // name -> Value with multiple versions + mutable folly::RWSpinLock lock_; std::unordered_map> valueMap_; }; diff --git a/src/graph/context/Symbols.cpp b/src/graph/context/Symbols.cpp index 8e2c1b60969..77602ed3f8d 100644 --- a/src/graph/context/Symbols.cpp +++ b/src/graph/context/Symbols.cpp @@ -22,6 +22,7 @@ std::string Variable::toString() const { } std::string SymbolTable::toString() const { + folly::RWSpinLock::ReadHolder holder(lock_); std::stringstream ss; ss << "SymTable: ["; for (const auto& p : vars_) { @@ -35,12 +36,19 @@ std::string SymbolTable::toString() const { } bool SymbolTable::existsVar(const std::string& varName) const { + folly::RWSpinLock::ReadHolder holder(lock_); return vars_.find(varName) != vars_.end(); } Variable* SymbolTable::newVariable(const std::string& name) { - VLOG(1) << "New variable for: " << name; - DCHECK(vars_.find(name) == vars_.end()); +#ifdef NDEBUG + { + // addVar has a write lock, should warp this read lock block + folly::RWSpinLock::ReadHolder holder(lock_); + VLOG(1) << "New variable for: " << name; + DCHECK(vars_.find(name) == vars_.end()); + } +#endif auto* variable = objPool_->makeAndAdd(name); addVar(name, variable); // Initialize all variable in variable map (output of node, inner variable etc.) @@ -50,10 +58,12 @@ Variable* SymbolTable::newVariable(const std::string& name) { } void SymbolTable::addVar(std::string varName, Variable* variable) { + folly::RWSpinLock::WriteHolder holder(lock_); vars_.emplace(std::move(varName), variable); } bool SymbolTable::readBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -63,6 +73,7 @@ bool SymbolTable::readBy(const std::string& varName, PlanNode* node) { } bool SymbolTable::writtenBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -72,6 +83,7 @@ bool SymbolTable::writtenBy(const std::string& varName, PlanNode* node) { } bool SymbolTable::deleteReadBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -81,6 +93,7 @@ bool SymbolTable::deleteReadBy(const std::string& varName, PlanNode* node) { } bool SymbolTable::deleteWrittenBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -102,6 +115,7 @@ bool SymbolTable::updateWrittenBy(const std::string& oldVar, } Variable* SymbolTable::getVar(const std::string& varName) { + folly::RWSpinLock::ReadHolder holder(lock_); DCHECK(!varName.empty()) << "the variable name is empty"; auto var = vars_.find(varName); if (var == vars_.end()) { diff --git a/src/graph/context/Symbols.h b/src/graph/context/Symbols.h index 84a497e11ba..3ced502d418 100644 --- a/src/graph/context/Symbols.h +++ b/src/graph/context/Symbols.h @@ -6,6 +6,7 @@ #ifndef GRAPH_CONTEXT_SYMBOLS_H_ #define GRAPH_CONTEXT_SYMBOLS_H_ +#include #include #include @@ -82,6 +83,7 @@ class SymbolTable final { ObjectPool* objPool_{nullptr}; ExecutionContext* ectx_{nullptr}; // var name -> variable + mutable folly::RWSpinLock lock_; std::unordered_map vars_; }; diff --git a/src/graph/executor/admin/SwitchSpaceExecutor.cpp b/src/graph/executor/admin/SwitchSpaceExecutor.cpp index d777742ab7e..f7a22d04fac 100644 --- a/src/graph/executor/admin/SwitchSpaceExecutor.cpp +++ b/src/graph/executor/admin/SwitchSpaceExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/SwitchSpaceExecutor.h" #include "clients/meta/MetaClient.h" +#include "common/memory/MemoryTracker.h" #include "graph/planner/plan/Query.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" @@ -13,6 +14,8 @@ namespace nebula { namespace graph { folly::Future SwitchSpaceExecutor::execute() { + // TODO: need check if this MemoryCheckOff is necessary + memory::MemoryCheckOffGuard guard; SCOPED_TIMER(&execTime_); auto *spaceToNode = asNode(node()); @@ -25,6 +28,11 @@ folly::Future SwitchSpaceExecutor::execute() { } auto spaceId = resp.value().get_space_id(); + // SwitchSpace can be mixed with normal query, if the corresponding query failed by other + // Executors, QueryContext may already be released. + if (!qctx() || !qctx()->rctx() || qctx_->rctx()->session() == nullptr) { + return Status::Error("Session not found"); + } auto *session = qctx_->rctx()->session(); NG_RETURN_IF_ERROR(PermissionManager::canReadSpace(session, spaceId)); const auto &properties = resp.value().get_properties(); diff --git a/src/graph/executor/algo/BFSShortestPathExecutor.cpp b/src/graph/executor/algo/BFSShortestPathExecutor.cpp index 098ad840df5..b098145fbe4 100644 --- a/src/graph/executor/algo/BFSShortestPathExecutor.cpp +++ b/src/graph/executor/algo/BFSShortestPathExecutor.cpp @@ -11,7 +11,6 @@ namespace nebula { namespace graph { folly::Future BFSShortestPathExecutor::execute() { // MemoryTrackerVerified - SCOPED_TIMER(&execTime_); pathNode_ = asNode(node()); terminateEarlyVar_ = pathNode_->terminateEarlyVar(); diff --git a/src/graph/gc/GC.cpp b/src/graph/gc/GC.cpp index de3428861a0..0d6b2899c52 100644 --- a/src/graph/gc/GC.cpp +++ b/src/graph/gc/GC.cpp @@ -23,12 +23,15 @@ GC::GC() { } void GC::clear(std::vector&& garbage) { + memory::MemoryCheckOffGuard guard; + // do not bother folly queue_.enqueue(std::move(garbage)); } void GC::periodicTask() { - // TODO: maybe could release by batch - queue_.try_dequeue(); + while (!queue_.empty()) { + queue_.try_dequeue(); + } } } // namespace graph } // namespace nebula diff --git a/src/graph/optimizer/rule/IndexScanRule.cpp b/src/graph/optimizer/rule/IndexScanRule.cpp index badab2b4ac6..bae416e36c8 100644 --- a/src/graph/optimizer/rule/IndexScanRule.cpp +++ b/src/graph/optimizer/rule/IndexScanRule.cpp @@ -144,7 +144,7 @@ Status IndexScanRule::createMultipleIQC(IndexQueryCtx& iqctx, return Status::OK(); } -size_t IndexScanRule::hintCount(const FilterItems& items) const noexcept { +size_t IndexScanRule::hintCount(const FilterItems& items) const { std::unordered_set hintCols; for (const auto& i : items.items) { hintCols.emplace(i.col_); diff --git a/src/graph/optimizer/rule/IndexScanRule.h b/src/graph/optimizer/rule/IndexScanRule.h index 8f8fb016c7d..2a334b84405 100644 --- a/src/graph/optimizer/rule/IndexScanRule.h +++ b/src/graph/optimizer/rule/IndexScanRule.h @@ -121,7 +121,7 @@ class IndexScanRule final : public OptRule { const FilterItems& items, const meta::cpp2::ColumnDef& col) const; - size_t hintCount(const FilterItems& items) const noexcept; + size_t hintCount(const FilterItems& items) const; bool isEdge(const OptGroupNode* groupNode) const; diff --git a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp index d2d121378bf..5c869a0ab0a 100644 --- a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp +++ b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp @@ -15,6 +15,19 @@ AsyncMsgNotifyBasedScheduler::AsyncMsgNotifyBasedScheduler(QueryContext* qctx) : query_ = qctx->rctx()->query(); } +void AsyncMsgNotifyBasedScheduler::waitFinish() { + std::unique_lock lck(emtx_); + cv_.wait(lck, [this] { + if (executing_ != 0) { + DLOG(INFO) << "executing: " << executing_; + return false; + } else { + DLOG(INFO) << " wait finish"; + return true; + } + }); +} + folly::Future AsyncMsgNotifyBasedScheduler::schedule() { auto root = qctx_->plan()->root(); if (FLAGS_enable_lifetime_optimize) { @@ -24,6 +37,7 @@ folly::Future AsyncMsgNotifyBasedScheduler::schedule() { analyzeLifetime(root); } auto executor = Executor::create(root, qctx_); + DLOG(INFO) << formatPrettyDependencyTree(executor); return doSchedule(executor); } @@ -90,7 +104,7 @@ folly::Future AsyncMsgNotifyBasedScheduler::doSchedule(Executor* root) c return folly::makeFuture(Status::GraphMemoryExceeded( "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); }) - .thenTry([this, pros = std::move(currentExePromises)](auto&& t) mutable { + .thenTry([this, exe, pros = std::move(currentExePromises)](auto&& t) mutable { // any exception or status not ok handled with notifyError if (t.hasException()) { notifyError(pros, Status::Error(std::move(t).exception().what())); @@ -99,6 +113,8 @@ folly::Future AsyncMsgNotifyBasedScheduler::doSchedule(Executor* root) c if (v.ok()) { notifyOK(pros); } else { + DLOG(INFO) << "[" << exe->name() << "," << exe->id() << "]" + << " fail with: " << v.toString(); notifyError(pros, v); } } @@ -143,7 +159,6 @@ folly::Future AsyncMsgNotifyBasedScheduler::runSelect( return execute(select); }) .thenValue([select, this](auto&& selectStatus) mutable -> folly::Future { - memory::MemoryCheckGuard guard; NG_RETURN_IF_ERROR(selectStatus); auto val = qctx_->ectx()->getValue(select->node()->outputVar()); if (!val.isBool()) { @@ -185,7 +200,6 @@ folly::Future AsyncMsgNotifyBasedScheduler::runLoop( return execute(loop); }) .thenValue([loop, runner, this](auto&& loopStatus) mutable -> folly::Future { - memory::MemoryCheckGuard guard; NG_RETURN_IF_ERROR(loopStatus); auto val = qctx_->ectx()->getValue(loop->node()->outputVar()); if (!val.isBool()) { @@ -225,17 +239,94 @@ void AsyncMsgNotifyBasedScheduler::notifyError(std::vector AsyncMsgNotifyBasedScheduler::execute(Executor* executor) const { - memory::MemoryCheckGuard guard1; auto status = executor->open(); if (!status.ok()) { return executor->error(std::move(status)); } - return executor->execute().thenValue([executor](Status s) { - memory::MemoryCheckGuard guard2; - NG_RETURN_IF_ERROR(s); - return executor->close(); + + auto exeStatus = runExecute(executor); + + return std::move(exeStatus).thenValue([this, executor](Status s) { + if (!s.ok()) { + DLOG(INFO) << formatPrettyId(executor) << " failed with: " << s.toString(); + setFailStatus(s); + removeExecuting(executor); + return Status::from(s); + } + auto ret = executor->close(); + removeExecuting(executor); + return ret; }); } +folly::Future AsyncMsgNotifyBasedScheduler::runExecute(Executor* executor) const { + // catch Executor::execute here, upward call stack should only get Status, no exceptions. + try { + addExecuting(executor); + if (hasFailStatus()) return failedStatus_.value(); + folly::Future status = Status::OK(); + { + memory::MemoryCheckGuard guard; + status = executor->execute(); + } + return std::move(status).thenError(folly::tag_t{}, [](const std::bad_alloc&) { + return folly::makeFuture(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + }); + } catch (std::bad_alloc& e) { + return folly::makeFuture(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + } catch (std::exception& e) { + return folly::makeFuture(Status::Error("%s", e.what())); + } catch (...) { + return folly::makeFuture(Status::Error("unknown error")); + } +} + +void AsyncMsgNotifyBasedScheduler::addExecuting(Executor* executor) const { + std::unique_lock lck(emtx_); + executing_++; + DLOG(INFO) << formatPrettyId(executor) << " add " << executing_; +} + +void AsyncMsgNotifyBasedScheduler::removeExecuting(Executor* executor) const { + std::unique_lock lck(emtx_); + executing_--; + DLOG(INFO) << formatPrettyId(executor) << "remove: " << executing_; + cv_.notify_one(); +} + +void AsyncMsgNotifyBasedScheduler::setFailStatus(Status status) const { + std::unique_lock lck(smtx_); + if (!failedStatus_.has_value()) { + failedStatus_ = status; + } +} + +bool AsyncMsgNotifyBasedScheduler::hasFailStatus() const { + std::unique_lock lck(smtx_); + return failedStatus_.has_value(); +} + +std::string AsyncMsgNotifyBasedScheduler::formatPrettyId(Executor* executor) { + return fmt::format("[{},{}]", executor->name(), executor->id()); +} + +std::string AsyncMsgNotifyBasedScheduler::formatPrettyDependencyTree(Executor* root) { + std::stringstream ss; + size_t spaces = 0; + appendExecutor(spaces, root, ss); + return ss.str(); +} + +void AsyncMsgNotifyBasedScheduler::appendExecutor(size_t spaces, + Executor* executor, + std::stringstream& ss) { + ss << std::string(spaces, ' ') << formatPrettyId(executor) << std::endl; + for (auto depend : executor->depends()) { + appendExecutor(spaces + 1, depend, ss); + } +} + } // namespace graph } // namespace nebula diff --git a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h index 5698d2fcce3..a9788fe8c15 100644 --- a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h +++ b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h @@ -27,6 +27,8 @@ class AsyncMsgNotifyBasedScheduler final : public Scheduler { folly::Future schedule() override; + void waitFinish() override; + private: folly::Future doSchedule(Executor* root) const; @@ -62,6 +64,29 @@ class AsyncMsgNotifyBasedScheduler final : public Scheduler { folly::Future execute(Executor* executor) const; + folly::Future runExecute(Executor* executor) const; + + void addExecuting(Executor* executor) const; + + void removeExecuting(Executor* executor) const; + + void setFailStatus(Status status) const; + + bool hasFailStatus() const; + + static std::string formatPrettyId(Executor* executor); + + static std::string formatPrettyDependencyTree(Executor* root); + + static void appendExecutor(size_t tabs, Executor* executor, std::stringstream& ss); + + private: + // set if some Executor failed, then other Executors no need to do Executor::execute() further + mutable std::mutex smtx_; + mutable std::optional failedStatus_; + mutable std::mutex emtx_; + mutable std::condition_variable cv_; + mutable size_t executing_{0}; QueryContext* qctx_{nullptr}; }; } // namespace graph diff --git a/src/graph/scheduler/Scheduler.h b/src/graph/scheduler/Scheduler.h index 43e765de292..34b13b8bbca 100644 --- a/src/graph/scheduler/Scheduler.h +++ b/src/graph/scheduler/Scheduler.h @@ -22,6 +22,8 @@ class Scheduler : private boost::noncopyable, private cpp::NonMovable { virtual folly::Future schedule() = 0; + virtual void waitFinish() = 0; + static void analyzeLifetime(const PlanNode *node, std::size_t loopLayers = 0); protected: diff --git a/src/graph/service/QueryEngine.cpp b/src/graph/service/QueryEngine.cpp index 4f6927dbcd1..f7045f7fd50 100644 --- a/src/graph/service/QueryEngine.cpp +++ b/src/graph/service/QueryEngine.cpp @@ -46,7 +46,6 @@ Status QueryEngine::init(std::shared_ptr ioExecutor // Create query context and query instance and execute it void QueryEngine::execute(RequestContextPtr rctx) { - memory::MemoryCheckGuard guard; auto qctx = std::make_unique(std::move(rctx), schemaManager_.get(), indexManager_.get(), diff --git a/src/graph/service/QueryInstance.cpp b/src/graph/service/QueryInstance.cpp index 9e4d9f6406c..cc1f6cd457b 100644 --- a/src/graph/service/QueryInstance.cpp +++ b/src/graph/service/QueryInstance.cpp @@ -38,7 +38,6 @@ QueryInstance::QueryInstance(std::unique_ptr qctx, Optimizer *opti void QueryInstance::execute() { try { - memory::MemoryCheckGuard guard1; Status status = validateAndOptimize(); if (!status.ok()) { onError(std::move(status)); @@ -55,7 +54,6 @@ void QueryInstance::execute() { // series of Executors through the Scheduler to drive the execution of the Executors. scheduler_->schedule() .thenValue([this](Status s) { - memory::MemoryCheckGuard guard2; if (s.ok()) { this->onFinish(); } else { @@ -139,6 +137,7 @@ void QueryInstance::onFinish() { rctx->finish(); rctx->session()->deleteQuery(qctx_.get()); + scheduler_->waitFinish(); // The `QueryInstance' is the root node holding all resources during the // execution. When the whole query process is done, it's safe to release this // object, as long as no other contexts have chances to access these resources diff --git a/src/graph/service/RequestContext.h b/src/graph/service/RequestContext.h index 80348480680..4f659ac96d6 100644 --- a/src/graph/service/RequestContext.h +++ b/src/graph/service/RequestContext.h @@ -55,7 +55,7 @@ class RequestContext final : public boost::noncopyable, public cpp::NonMovable { } void setSession(std::shared_ptr session) { - session_ = std::move(session); + session_ = session; if (session_ != nullptr) { // Keep the session active session_->charge(); diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index eadacf20092..94bab5723eb 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -143,6 +143,7 @@ nebula::cpp2::ErrorCode RocksEngine::commitBatchWrite(std::unique_ptr(snapshot); @@ -161,6 +162,7 @@ nebula::cpp2::ErrorCode RocksEngine::get(const std::string& key, std::vector RocksEngine::multiGet(const std::vector& keys, std::vector* values) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; std::vector slices; for (size_t index = 0; index < keys.size(); index++) { @@ -184,23 +186,25 @@ std::vector RocksEngine::multiGet(const std::vector& keys, nebula::cpp2::ErrorCode RocksEngine::range(const std::string& start, const std::string& end, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (!isPlainTable_) { options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; } else { options.prefix_same_as_start = true; } - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(start)); } - storageIter->reset(new RocksRangeIter(iter, start, end)); + storageIter->reset(new RocksRangeIter(std::move(iter), start, end)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::prefix(const std::string& prefix, std::unique_ptr* storageIter, const void* snapshot) { + memory::MemoryCheckOffGuard guard; // In fact, we don't need to check prefix.size() >= extractorLen_, which is caller's duty to make // sure the prefix bloom filter exists. But this is quite error-prone, so we do a check here. if (FLAGS_enable_rocksdb_prefix_filtering && prefix.size() >= extractorLen_) { @@ -213,58 +217,62 @@ nebula::cpp2::ErrorCode RocksEngine::prefix(const std::string& prefix, nebula::cpp2::ErrorCode RocksEngine::prefixWithExtractor(const std::string& prefix, const void* snapshot, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (UNLIKELY(snapshot != nullptr)) { options.snapshot = reinterpret_cast(snapshot); } options.prefix_same_as_start = true; - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(prefix)); } - storageIter->reset(new RocksPrefixIter(iter, prefix)); + storageIter->reset(new RocksPrefixIter(std::move(iter), prefix)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::prefixWithoutExtractor( const std::string& prefix, const void* snapshot, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (snapshot != nullptr) { options.snapshot = reinterpret_cast(snapshot); } // prefix_same_as_start is false by default options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(prefix)); } - storageIter->reset(new RocksPrefixIter(iter, prefix)); + storageIter->reset(new RocksPrefixIter(std::move(iter), prefix)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::rangeWithPrefix(const std::string& start, const std::string& prefix, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (!isPlainTable_) { options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; } else { options.prefix_same_as_start = true; } - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(start)); } - storageIter->reset(new RocksPrefixIter(iter, prefix)); + storageIter->reset(new RocksPrefixIter(std::move(iter), prefix)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::scan(std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; options.total_order_seek = true; - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); iter->SeekToFirst(); - storageIter->reset(new RocksCommonIter(iter)); + storageIter->reset(new RocksCommonIter(std::move(iter))); return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index e4be3f1a5b9..1a529686ffd 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -11,6 +11,8 @@ #include #include +#include + #include "common/base/Base.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" @@ -27,6 +29,9 @@ class RocksRangeIter : public KVIterator { RocksRangeIter(rocksdb::Iterator* iter, rocksdb::Slice start, rocksdb::Slice end) : iter_(iter), start_(start), end_(end) {} + RocksRangeIter(std::unique_ptr iter, rocksdb::Slice start, rocksdb::Slice end) + : iter_(std::move(iter)), start_(start), end_(end) {} + ~RocksRangeIter() = default; bool valid() const override { @@ -62,6 +67,9 @@ class RocksPrefixIter : public KVIterator { public: RocksPrefixIter(rocksdb::Iterator* iter, rocksdb::Slice prefix) : iter_(iter), prefix_(prefix) {} + RocksPrefixIter(std::unique_ptr iter, rocksdb::Slice prefix) + : iter_(std::move(iter)), prefix_(prefix) {} + ~RocksPrefixIter() = default; bool valid() const override { @@ -96,6 +104,8 @@ class RocksCommonIter : public KVIterator { public: explicit RocksCommonIter(rocksdb::Iterator* iter) : iter_(iter) {} + explicit RocksCommonIter(std::unique_ptr iter) : iter_(std::move(iter)) {} + ~RocksCommonIter() = default; bool valid() const override { diff --git a/src/kvstore/listener/elasticsearch/ESListener.cpp b/src/kvstore/listener/elasticsearch/ESListener.cpp index 92435e6ec22..1f40cba4a22 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.cpp +++ b/src/kvstore/listener/elasticsearch/ESListener.cpp @@ -220,9 +220,7 @@ bool ESListener::writeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLo return true; } -std::string ESListener::encodeAppliedId(LogID lastId, - TermID lastTerm, - LogID lastApplyLogId) const noexcept { +std::string ESListener::encodeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLogId) const { std::string val; val.reserve(sizeof(LogID) * 2 + sizeof(TermID)); val.append(reinterpret_cast(&lastId), sizeof(LogID)) diff --git a/src/kvstore/listener/elasticsearch/ESListener.h b/src/kvstore/listener/elasticsearch/ESListener.h index b46cc58f830..4edbb30437e 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.h +++ b/src/kvstore/listener/elasticsearch/ESListener.h @@ -102,7 +102,7 @@ class ESListener : public Listener { * @param lastApplyLogId Last apply id * @return Encoded string */ - std::string encodeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLogId) const noexcept; + std::string encodeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLogId) const; private: meta::SchemaManager* schemaMan_{nullptr}; diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index ff3909b3970..7d66e518b76 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -87,16 +87,14 @@ class AppendLogsIterator final : public LogIterator { ~AppendLogsIterator() { if (!logs_.empty()) { - size_t notFulfilledPromise = 0; for (auto& log : logs_) { auto& promiseRef = std::get<4>(log); if (!promiseRef.isFulfilled()) { - ++notFulfilledPromise; + // When a AppendLogsIterator destruct before calling commit, the promise it not fulfilled. + // It only happens when exception is thrown, which make `commit` is never invoked. + promiseRef.setValue(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED); } } - if (notFulfilledPromise > 0) { - LOG(FATAL) << "notFulfilledPromise == " << notFulfilledPromise; - } } } diff --git a/src/storage/BaseProcessor.h b/src/storage/BaseProcessor.h index b94fc723f08..1721be49cf3 100644 --- a/src/storage/BaseProcessor.h +++ b/src/storage/BaseProcessor.h @@ -39,6 +39,7 @@ class BaseProcessor { } virtual void onFinished() { + memory::MemoryCheckOffGuard guard; if (counters_) { stats::StatsManager::addValue(counters_->numCalls_); if (!this->result_.get_failed_parts().empty()) { @@ -62,6 +63,7 @@ class BaseProcessor { } virtual void onError() { + memory::MemoryCheckOffGuard guard; if (counters_) { stats::StatsManager::addValue(counters_->numCalls_); if (!this->result_.get_failed_parts().empty()) { @@ -171,7 +173,6 @@ struct MemoryCheckScope { : processor_(processor), f_(std::move(f)) {} ~MemoryCheckScope() { - memory::MemoryCheckGuard guard; try { f_(); } catch (std::bad_alloc& e) { diff --git a/src/storage/GraphStorageServiceHandler.cpp b/src/storage/GraphStorageServiceHandler.cpp index 7d0d18971d5..72c386148e0 100644 --- a/src/storage/GraphStorageServiceHandler.cpp +++ b/src/storage/GraphStorageServiceHandler.cpp @@ -31,20 +31,20 @@ // Processors DO NOT NEED handle error in their logic. // else (do some work in another thread) // Processors need handle error in that thread by itself -#define RETURN_FUTURE(processor) \ - memory::MemoryCheckGuard guard; \ - auto f = processor->getFuture(); \ - try { \ - processor->process(req); \ - } catch (std::bad_alloc & e) { \ - processor->memoryExceeded(); \ - processor->onError(); \ - } catch (std::exception & e) { \ - LOG(ERROR) << e.what(); \ - processor->onError(); \ - } catch (...) { \ - processor->onError(); \ - } \ +#define RETURN_FUTURE(processor) \ + auto f = processor->getFuture(); \ + try { \ + processor->process(req); \ + } catch (std::bad_alloc & e) { \ + LOG(ERROR) << processor << " bad_alloc"; \ + processor->memoryExceeded(); \ + processor->onError(); \ + } catch (std::exception & e) { \ + LOG(ERROR) << e.what(); \ + processor->onError(); \ + } catch (...) { \ + processor->onError(); \ + } \ return f; namespace nebula { diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 975cc348794..46b79c6be4d 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -218,6 +218,7 @@ ErrorOr> LookupProcessor::bu void LookupProcessor::runInSingleThread(const std::vector& parts, std::unique_ptr plan) { + memory::MemoryCheckGuard guard; // printPlan(plan.get()); std::vector> datasetList; std::vector<::nebula::cpp2::ErrorCode> codeList; @@ -265,6 +266,7 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, void LookupProcessor::runInMultipleThread(const std::vector& parts, std::unique_ptr plan) { + memory::MemoryCheckOffGuard offGuard; std::vector> planCopy = reproducePlan(plan.get(), parts.size()); using ReturnType = std::tuple, Row>; std::vector> futures; @@ -298,49 +300,47 @@ void LookupProcessor::runInMultipleThread(const std::vector& parts, } return {part, code, dataset, statResult}; }) - .thenError( - folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture< - std::tuple, Row>>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; return folly::makeFuture< std::tuple, Row>>( - std::runtime_error(e.what())); + std::runtime_error("Memory Limit Exceeded, " + + memory::MemoryStats::instance().toString())); })); } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - std::vector statResults; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { - onError(); - return; - } - auto& [partId, code, dataset, statResult] = tries[j].value(); - if (code == ::nebula::cpp2::ErrorCode::SUCCEEDED) { - for (auto& row : dataset) { - resultDataSet_.emplace_back(std::move(row)); + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + std::vector statResults; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + auto& [partId, code, dataset, statResult] = tries[j].value(); + if (code == ::nebula::cpp2::ErrorCode::SUCCEEDED) { + for (auto& row : dataset) { + resultDataSet_.emplace_back(std::move(row)); + } + } else { + handleErrorCode(code, context_->spaceId(), partId); + } + statResults.emplace_back(std::move(statResult)); } - } else { - handleErrorCode(code, context_->spaceId(), partId); - } - statResults.emplace_back(std::move(statResult)); - } - DLOG(INFO) << "finish"; - // IndexAggregateNode has been copied and each part get it's own aggregate info, - // we need to merge it - this->mergeStatsResult(statResults); - this->onProcessFinished(); - this->onFinished(); - }); + DLOG(INFO) << "finish"; + // IndexAggregateNode has been copied and each part get it's own aggregate info, + // we need to merge it + this->mergeStatsResult(statResults); + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; + onError(); + }); } ErrorOr>> diff --git a/src/storage/query/GetDstBySrcProcessor.cpp b/src/storage/query/GetDstBySrcProcessor.cpp index f4a1ce38029..b6ee5f2a4a2 100644 --- a/src/storage/query/GetDstBySrcProcessor.cpp +++ b/src/storage/query/GetDstBySrcProcessor.cpp @@ -66,6 +66,7 @@ void GetDstBySrcProcessor::doProcess(const cpp2::GetDstBySrcRequest& req) { } void GetDstBySrcProcessor::runInSingleThread(const cpp2::GetDstBySrcRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); auto plan = buildPlan(&contexts_.front(), &flatResult_); std::unordered_set failedParts; @@ -101,6 +102,7 @@ void GetDstBySrcProcessor::runInSingleThread(const cpp2::GetDstBySrcRequest& req } void GetDstBySrcProcessor::runInMultipleThread(const cpp2::GetDstBySrcRequest& req) { + memory::MemoryCheckOffGuard offGuard; for (size_t i = 0; i < req.get_parts().size(); i++) { partResults_.emplace_back(); contexts_.emplace_back(RuntimeContext(planContext_.get())); @@ -112,39 +114,45 @@ void GetDstBySrcProcessor::runInMultipleThread(const cpp2::GetDstBySrcRequest& r i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - - // size_t sum = 0; - // for (size_t j = 0; j < tries.size(); j++) { - // const auto& [code, partId] = tries[j].value(); - // if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { - // sum += partResults_[j].size(); - // } - // } - // flatResult_.reserve(sum); - - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { - onError(); - return; - } - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - for (auto& v : partResults_[j]) { - flatResult_.emplace_back(std::move(v)); + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + + // size_t sum = 0; + // for (size_t j = 0; j < tries.size(); j++) { + // const auto& [code, partId] = tries[j].value(); + // if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { + // sum += partResults_[j].size(); + // } + // } + // flatResult_.reserve(sum); + + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + for (auto& v : partResults_[j]) { + flatResult_.emplace_back(std::move(v)); + } + std::deque().swap(partResults_[j]); + } } - std::deque().swap(partResults_[j]); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; + onError(); + }); } folly::Future> GetDstBySrcProcessor::runInExecutor( @@ -155,6 +163,10 @@ folly::Future> GetDstBySrcProces return folly::via(executor_, [this, context, result, partId, input = std::move(srcIds)]() mutable { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, + partId); + } auto plan = buildPlan(context, result); for (const auto& src : input) { auto& vId = src.getStr(); @@ -176,17 +188,9 @@ folly::Future> GetDstBySrcProces } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } diff --git a/src/storage/query/GetNeighborsProcessor.cpp b/src/storage/query/GetNeighborsProcessor.cpp index 07c68d6a62a..b362ee6d60a 100644 --- a/src/storage/query/GetNeighborsProcessor.cpp +++ b/src/storage/query/GetNeighborsProcessor.cpp @@ -77,6 +77,7 @@ void GetNeighborsProcessor::doProcess(const cpp2::GetNeighborsRequest& req) { void GetNeighborsProcessor::runInSingleThread(const cpp2::GetNeighborsRequest& req, int64_t limit, bool random) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); auto plan = buildPlan(&contexts_.front(), &expCtxs_.front(), &resultDataSet_, limit, random); @@ -115,6 +116,7 @@ void GetNeighborsProcessor::runInSingleThread(const cpp2::GetNeighborsRequest& r void GetNeighborsProcessor::runInMultipleThread(const cpp2::GetNeighborsRequest& req, int64_t limit, bool random) { + memory::MemoryCheckOffGuard offGuard; for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; results_.emplace_back(std::move(result)); @@ -129,30 +131,36 @@ void GetNeighborsProcessor::runInMultipleThread(const cpp2::GetNeighborsRequest& i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } folly::Future> GetNeighborsProcessor::runInExecutor( @@ -167,6 +175,9 @@ folly::Future> GetNeighborsProce executor_, [this, context, expCtx, result, partId, input = std::move(vids), limit, random]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); + } auto plan = buildPlan(context, expCtx, result, limit, random); for (const auto& vid : input) { auto vId = vid.getStr(); @@ -188,17 +199,9 @@ folly::Future> GetNeighborsProce } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } diff --git a/src/storage/query/GetPropProcessor.cpp b/src/storage/query/GetPropProcessor.cpp index 81dfb5ef7e2..b5dfaf35437 100644 --- a/src/storage/query/GetPropProcessor.cpp +++ b/src/storage/query/GetPropProcessor.cpp @@ -56,6 +56,7 @@ void GetPropProcessor::doProcess(const cpp2::GetPropRequest& req) { } void GetPropProcessor::runInSingleThread(const cpp2::GetPropRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); std::unordered_set failedParts; if (!isEdge_) { @@ -116,6 +117,7 @@ void GetPropProcessor::runInSingleThread(const cpp2::GetPropRequest& req) { } void GetPropProcessor::runInMultipleThread(const cpp2::GetPropRequest& req) { + memory::MemoryCheckOffGuard offGuard; for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; results_.emplace_back(std::move(result)); @@ -128,30 +130,36 @@ void GetPropProcessor::runInMultipleThread(const cpp2::GetPropRequest& req) { i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } folly::Future> GetPropProcessor::runInExecutor( @@ -162,6 +170,10 @@ folly::Future> GetPropProcessor: return folly::via(executor_, [this, context, result, partId, input = std::move(rows)]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, + partId); + } if (!isEdge_) { auto plan = buildTagPlan(context, result); for (const auto& row : input) { @@ -206,17 +218,11 @@ folly::Future> GetPropProcessor: return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); } }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; return folly::makeFuture>( - std::runtime_error(e.what())); + std::runtime_error("Memory Limit Exceeded, " + + memory::MemoryStats::instance().toString())); }); } diff --git a/src/storage/query/ScanEdgeProcessor.cpp b/src/storage/query/ScanEdgeProcessor.cpp index c49d72906f5..a193ad7da7c 100644 --- a/src/storage/query/ScanEdgeProcessor.cpp +++ b/src/storage/query/ScanEdgeProcessor.cpp @@ -128,6 +128,10 @@ folly::Future> ScanEdgeProcessor return folly::via(executor_, [this, context, result, cursors, partId, input = std::move(cursor), expCtx]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, + partId); + } auto plan = buildPlan(context, result, cursors, expCtx); auto ret = plan.go(partId, input); @@ -136,21 +140,14 @@ folly::Future> ScanEdgeProcessor } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } void ScanEdgeProcessor::runInSingleThread(const cpp2::ScanEdgeRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); std::unordered_set failedParts; @@ -172,6 +169,7 @@ void ScanEdgeProcessor::runInSingleThread(const cpp2::ScanEdgeRequest& req) { } void ScanEdgeProcessor::runInMultipleThread(const cpp2::ScanEdgeRequest& req) { + memory::MemoryCheckOffGuard offGuard; cursorsOfPart_.resize(req.get_parts().size()); for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; @@ -192,31 +190,37 @@ void ScanEdgeProcessor::runInMultipleThread(const cpp2::ScanEdgeRequest& req) { i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + cursors_.merge(std::move(cursorsOfPart_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - cursors_.merge(std::move(cursorsOfPart_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } } // namespace storage diff --git a/src/storage/query/ScanVertexProcessor.cpp b/src/storage/query/ScanVertexProcessor.cpp index 6465b11de4b..cb5754aa527 100644 --- a/src/storage/query/ScanVertexProcessor.cpp +++ b/src/storage/query/ScanVertexProcessor.cpp @@ -133,6 +133,9 @@ folly::Future> ScanVertexProcess executor_, [this, context, result, cursorsOfPart, partId, input = std::move(cursor), expCtx]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); + } auto plan = buildPlan(context, result, cursorsOfPart, expCtx); auto ret = plan.go(partId, input); @@ -141,21 +144,14 @@ folly::Future> ScanVertexProcess } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } void ScanVertexProcessor::runInSingleThread(const cpp2::ScanVertexRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); std::unordered_set failedParts; @@ -177,6 +173,7 @@ void ScanVertexProcessor::runInSingleThread(const cpp2::ScanVertexRequest& req) } void ScanVertexProcessor::runInMultipleThread(const cpp2::ScanVertexRequest& req) { + memory::MemoryCheckOffGuard offGuard; cursorsOfPart_.resize(req.get_parts().size()); for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; @@ -197,31 +194,37 @@ void ScanVertexProcessor::runInMultipleThread(const cpp2::ScanVertexRequest& req i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + cursors_.merge(std::move(cursorsOfPart_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - cursors_.merge(std::move(cursorsOfPart_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } } // namespace storage