From 115434dc5fbb3da10625e47e7ad9a27b65d9318a Mon Sep 17 00:00:00 2001 From: Yang <1580680232@qq.com> Date: Fri, 22 Oct 2021 15:11:40 +0800 Subject: [PATCH] lookup support push down topK --- src/clients/storage/GraphStorageClient.cpp | 7 +- src/clients/storage/GraphStorageClient.h | 3 +- src/common/utils/TopKHeap.h | 71 ++++++ src/common/utils/test/CMakeLists.txt | 10 + src/common/utils/test/TopKHeapTest.cpp | 50 ++++ src/graph/context/ast/QueryAstContext.h | 1 + .../executor/query/IndexScanExecutor.cpp | 1 + src/graph/executor/query/TopNExecutor.cpp | 22 +- src/graph/optimizer/CMakeLists.txt | 11 +- src/graph/optimizer/OptimizerUtils.cpp | 1 + .../PushLimitDownEdgeIndexFullScanRule.cpp | 80 ------- .../rule/PushLimitDownEdgeIndexFullScanRule.h | 32 --- .../PushLimitDownEdgeIndexPrefixScanRule.cpp | 82 ------- .../PushLimitDownEdgeIndexPrefixScanRule.h | 32 --- .../PushLimitDownEdgeIndexRangeScanRule.cpp | 80 ------- .../PushLimitDownEdgeIndexRangeScanRule.h | 32 --- .../optimizer/rule/PushLimitDownIndexRule.cpp | 146 ++++++++++++ .../optimizer/rule/PushLimitDownIndexRule.h | 100 ++++++++ .../rule/PushLimitDownIndexScanRule.cpp | 76 ------ .../rule/PushLimitDownProjectRule.cpp | 2 + .../PushLimitDownTagIndexFullScanRule.cpp | 79 ------- .../rule/PushLimitDownTagIndexFullScanRule.h | 32 --- .../PushLimitDownTagIndexPrefixScanRule.cpp | 80 ------- .../PushLimitDownTagIndexPrefixScanRule.h | 32 --- .../PushLimitDownTagIndexRangeScanRule.cpp | 80 ------- .../rule/PushLimitDownTagIndexRangeScanRule.h | 32 --- .../rule/PushLimitSortDownIndexScanRule.cpp | 58 +++++ .../rule/PushLimitSortDownIndexScanRule.h | 33 +++ .../optimizer/rule/PushTopNDownIndexRule.cpp | 176 ++++++++++++++ .../optimizer/rule/PushTopNDownIndexRule.h | 100 ++++++++ .../rule/PushTopNDownProjectRule.cpp | 73 ++++++ ...exScanRule.h => PushTopNDownProjectRule.h} | 4 +- src/graph/optimizer/rule/TopNRule.cpp | 6 - src/graph/planner/ngql/LookupPlanner.cpp | 2 + src/graph/planner/plan/Query.cpp | 2 + src/graph/planner/plan/Query.h | 21 +- src/graph/util/ToJson.cpp | 3 + src/graph/validator/LookupValidator.cpp | 13 + src/graph/validator/LookupValidator.h | 1 + src/interface/storage.thrift | 3 + src/storage/exec/TopKNode.h | 71 ++++++ src/storage/index/LookupBaseProcessor-inl.h | 30 ++- src/storage/index/LookupBaseProcessor.h | 4 +- src/storage/index/LookupProcessor.cpp | 3 + src/storage/test/IndexScanLimitTest.cpp | 20 +- src/storage/test/StorageDAGTest.cpp | 2 +- tests/common/plan_differ.py | 11 +- tests/tck/features/lookup/LookUpTopN.feature | 222 ++++++++++++++++++ 48 files changed, 1223 insertions(+), 809 deletions(-) create mode 100644 src/common/utils/TopKHeap.h create mode 100644 src/common/utils/test/TopKHeapTest.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.h delete mode 100644 src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.h delete mode 100644 src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.h create mode 100644 src/graph/optimizer/rule/PushLimitDownIndexRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitDownIndexRule.h delete mode 100644 src/graph/optimizer/rule/PushLimitDownIndexScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.h delete mode 100644 src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.h delete mode 100644 src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.cpp delete mode 100644 src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.h create mode 100644 src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.cpp create mode 100644 src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.h create mode 100644 src/graph/optimizer/rule/PushTopNDownIndexRule.cpp create mode 100644 src/graph/optimizer/rule/PushTopNDownIndexRule.h create mode 100644 src/graph/optimizer/rule/PushTopNDownProjectRule.cpp rename src/graph/optimizer/rule/{PushLimitDownIndexScanRule.h => PushTopNDownProjectRule.h} (88%) create mode 100644 src/storage/exec/TopKNode.h create mode 100644 tests/tck/features/lookup/LookUpTopN.feature diff --git a/src/clients/storage/GraphStorageClient.cpp b/src/clients/storage/GraphStorageClient.cpp index 0ffeb31e5ac..dab14c412f5 100644 --- a/src/clients/storage/GraphStorageClient.cpp +++ b/src/clients/storage/GraphStorageClient.cpp @@ -488,6 +488,7 @@ StorageRpcRespFuture GraphStorageClient::lookupIndex( bool isEdge, int32_t tagOrEdge, const std::vector& returnCols, + const std::vector& orderBy, int64_t limit) { // TODO(sky) : instead of isEdge and tagOrEdge to nebula::cpp2::SchemaID for graph layer. auto space = param.space; @@ -516,9 +517,13 @@ StorageRpcRespFuture GraphStorageClient::lookupIndex( cpp2::IndexSpec spec; spec.set_contexts(contexts); spec.set_schema_id(schemaId); + if (!orderBy.empty()) { + spec.set_order_by(orderBy); + } + spec.set_limit(limit); + req.set_indices(spec); req.set_common(common); - req.set_limit(limit); } return collectResponse( diff --git a/src/clients/storage/GraphStorageClient.h b/src/clients/storage/GraphStorageClient.h index 80f25ec3bbc..81fd76b53e0 100644 --- a/src/clients/storage/GraphStorageClient.h +++ b/src/clients/storage/GraphStorageClient.h @@ -126,7 +126,8 @@ class GraphStorageClient : public StorageClientBase& returnCols, - int64_t limit); + const std::vector& orderBy = std::vector(), + int64_t limit = std::numeric_limits::max()); StorageRpcRespFuture lookupAndTraverse( const CommonRequestParam& param, cpp2::IndexSpec indexSpec, cpp2::TraverseSpec traverseSpec); diff --git a/src/common/utils/TopKHeap.h b/src/common/utils/TopKHeap.h new file mode 100644 index 00000000000..7288d55a922 --- /dev/null +++ b/src/common/utils/TopKHeap.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef COMMON_UTILS_TOPKHEAP_H_ +#define COMMON_UTILS_TOPKHEAP_H_ + +#include "common/base/Base.h" + +namespace nebula { + +template +class TopKHeap final { + public: + TopKHeap(int heapSize, std::function comparator) + : heapSize_(heapSize), comparator_(std::move(comparator)) { + v_.reserve(heapSize); + } + ~TopKHeap() = default; + + void push(T data) { + if (v_.size() < static_cast(heapSize_)) { + v_.push_back(data); + adjustUp(v_.size() - 1); + return; + } + if (comparator_(data, v_[0])) { + v_[0] = data; + adjustDown(0); + } + } + + std::vector moveTopK() { return std::move(v_); } + + private: + void adjustDown(size_t parent) { + size_t child = parent * 2 + 1; + size_t size = v_.size(); + while (child < size) { + if (child + 1 < size && comparator_(v_[child], v_[child + 1])) { + child += 1; + } + if (!comparator_(v_[parent], v_[child])) { + return; + } + std::swap(v_[parent], v_[child]); + parent = child; + child = parent * 2 + 1; + } + } + void adjustUp(size_t child) { + size_t parent = (child - 1) >> 1; + while (0 != child) { + if (!comparator_(v_[parent], v_[child])) { + return; + } + std::swap(v_[parent], v_[child]); + child = parent; + parent = (child - 1) >> 1; + } + } + + private: + int heapSize_; + std::vector v_; + std::function comparator_; +}; +} // namespace nebula +#endif // COMMON_UTILS_TOPKHEAP_H_ diff --git a/src/common/utils/test/CMakeLists.txt b/src/common/utils/test/CMakeLists.txt index 5e78743cf97..a20cf1c2650 100644 --- a/src/common/utils/test/CMakeLists.txt +++ b/src/common/utils/test/CMakeLists.txt @@ -80,5 +80,15 @@ nebula_add_test( LIBRARIES ${THRIFT_LIBRARIES} ${PROXYGEN_LIBRARIES} +) + +nebula_add_test( + NAME + topk_heap_test + SOURCES + TopKHeapTest.cpp + OBJECTS + $ + LIBRARIES gtest ) diff --git a/src/common/utils/test/TopKHeapTest.cpp b/src/common/utils/test/TopKHeapTest.cpp new file mode 100644 index 00000000000..c07acd8b8b3 --- /dev/null +++ b/src/common/utils/test/TopKHeapTest.cpp @@ -0,0 +1,50 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include + +#include "common/utils/TopKHeap.h" + +namespace nebula { + +std::vector getVector(std::function comparator) { + TopKHeap topKHeap(3, comparator); + topKHeap.push(3); + topKHeap.push(6); + topKHeap.push(1); + topKHeap.push(9); + topKHeap.push(2); + topKHeap.push(7); + return topKHeap.moveTopK(); +} + +TEST(TopKHeapTest, minHeapTest) { + auto topK = getVector([](const int& lhs, const int& rhs) { return lhs < rhs; }); + std::stringstream ss; + for (auto& item : topK) { + ss << item << " "; + } + ASSERT_EQ(ss.str(), "3 2 1 "); +} + +TEST(TopKHeapTest, maxHeapTest) { + auto topK = getVector([](const int& lhs, const int& rhs) { return lhs > rhs; }); + std::stringstream ss; + for (auto& item : topK) { + ss << item << " "; + } + ASSERT_EQ(ss.str(), "6 7 9 "); +} + +} // namespace nebula + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::init(&argc, &argv, true); + google::SetStderrLogging(google::INFO); + + return RUN_ALL_TESTS(); +} diff --git a/src/graph/context/ast/QueryAstContext.h b/src/graph/context/ast/QueryAstContext.h index df6f7049529..98ef20ab986 100644 --- a/src/graph/context/ast/QueryAstContext.h +++ b/src/graph/context/ast/QueryAstContext.h @@ -117,6 +117,7 @@ struct LookupContext final : public AstContext { YieldColumns* yieldExpr{nullptr}; std::vector idxReturnCols; std::vector idxColNames; + std::unordered_map idxOutColsToReturnColsMap_; // order by }; diff --git a/src/graph/executor/query/IndexScanExecutor.cpp b/src/graph/executor/query/IndexScanExecutor.cpp index aefda16b84b..c145231f19c 100644 --- a/src/graph/executor/query/IndexScanExecutor.cpp +++ b/src/graph/executor/query/IndexScanExecutor.cpp @@ -46,6 +46,7 @@ folly::Future IndexScanExecutor::indexScan() { lookup->isEdge(), lookup->schemaId(), lookup->returnColumns(), + lookup->orderBy(), lookup->limit()) .via(runner()) .thenValue([this](StorageRpcResponse &&rpcResp) { diff --git a/src/graph/executor/query/TopNExecutor.cpp b/src/graph/executor/query/TopNExecutor.cpp index ca3c3dc798e..e742a02035b 100644 --- a/src/graph/executor/query/TopNExecutor.cpp +++ b/src/graph/executor/query/TopNExecutor.cpp @@ -7,6 +7,7 @@ #include "graph/executor/query/TopNExecutor.h" #include "common/time/ScopedTimer.h" +#include "common/utils/TopKHeap.h" #include "graph/planner/plan/Query.h" namespace nebula { @@ -20,7 +21,7 @@ folly::Future TopNExecutor::execute() { if (UNLIKELY(iter == nullptr)) { return Status::Error("Internal error: nullptr iterator in topn executor"); } - if (UNLIKELY(!result.iter()->isSequentialIter())) { + if (UNLIKELY(!result.iter()->isSequentialIter() && !result.iter()->isPropIter())) { std::stringstream ss; ss << "Internal error: Sort executor does not supported " << iter->kind(); LOG(ERROR) << ss.str(); @@ -71,23 +72,18 @@ folly::Future TopNExecutor::execute() { template void TopNExecutor::executeTopN(Iterator *iter) { auto uIter = static_cast(iter); - std::vector heap(uIter->begin(), uIter->begin() + heapSize_); - std::make_heap(heap.begin(), heap.end(), comparator_); - auto it = uIter->begin() + heapSize_; + auto it = uIter->begin(); + TopKHeap topKHeap(heapSize_, comparator_); while (it != uIter->end()) { - if (comparator_(*it, heap[0])) { - std::pop_heap(heap.begin(), heap.end(), comparator_); - heap.pop_back(); - heap.push_back(*it); - std::push_heap(heap.begin(), heap.end(), comparator_); - } - ++it; + topKHeap.push(*it); + it++; } - std::sort_heap(heap.begin(), heap.end(), comparator_); + std::vector topK = topKHeap.moveTopK(); + std::sort_heap(topK.begin(), topK.end(), comparator_); auto beg = uIter->begin(); for (int i = 0; i < maxCount_; ++i) { - beg[i] = heap[offset_ + i]; + beg[i] = topK[offset_ + i]; } } diff --git a/src/graph/optimizer/CMakeLists.txt b/src/graph/optimizer/CMakeLists.txt index 361463cd882..7c36df308e3 100644 --- a/src/graph/optimizer/CMakeLists.txt +++ b/src/graph/optimizer/CMakeLists.txt @@ -38,14 +38,11 @@ nebula_add_library( rule/IndexFullScanBaseRule.cpp rule/TagIndexFullScanRule.cpp rule/EdgeIndexFullScanRule.cpp - rule/PushLimitDownIndexScanRule.cpp - rule/PushLimitDownTagIndexFullScanRule.cpp - rule/PushLimitDownTagIndexPrefixScanRule.cpp - rule/PushLimitDownTagIndexRangeScanRule.cpp - rule/PushLimitDownEdgeIndexFullScanRule.cpp - rule/PushLimitDownEdgeIndexPrefixScanRule.cpp - rule/PushLimitDownEdgeIndexRangeScanRule.cpp rule/PushLimitDownProjectRule.cpp + rule/PushTopNDownProjectRule.cpp + rule/PushLimitSortDownIndexScanRule.cpp + rule/PushTopNDownIndexRule.cpp + rule/PushLimitDownIndexRule.cpp ) nebula_add_subdirectory(test) diff --git a/src/graph/optimizer/OptimizerUtils.cpp b/src/graph/optimizer/OptimizerUtils.cpp index f2046707f63..9075a730e08 100644 --- a/src/graph/optimizer/OptimizerUtils.cpp +++ b/src/graph/optimizer/OptimizerUtils.cpp @@ -953,6 +953,7 @@ void OptimizerUtils::copyIndexScanData(const nebula::graph::IndexScan* from, to->setEmptyResultSet(from->isEmptyResultSet()); to->setSpace(from->space()); to->setReturnCols(from->returnColumns()); + to->setOutPutColsToReturnCols(from->outPutColsToReturnCols()); to->setIsEdge(from->isEdge()); to->setSchemaId(from->schemaId()); to->setDedup(from->dedup()); diff --git a/src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.cpp deleted file mode 100644 index 8fe76203b79..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Scan.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::EdgeIndexFullScan; -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::QueryContext; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownEdgeIndexFullScanRule::kInstance = - std::unique_ptr(new PushLimitDownEdgeIndexFullScanRule()); - -PushLimitDownEdgeIndexFullScanRule::PushLimitDownEdgeIndexFullScanRule() { - RuleSet::QueryRules().addRule(this); -} - -const Pattern &PushLimitDownEdgeIndexFullScanRule::pattern() const { - static Pattern pattern = Pattern::create( - graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexFullScan)}); - return pattern; -} - -StatusOr PushLimitDownEdgeIndexFullScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newEdgeIndexFullScan = static_cast(indexScan->clone()); - newEdgeIndexFullScan->setLimit(limitRows); - auto newEdgeIndexFullScanGroup = OptGroup::create(octx); - auto newEdgeIndexFullScanGroupNode = - newEdgeIndexFullScanGroup->makeGroupNode(newEdgeIndexFullScan); - - newLimitGroupNode->dependsOn(newEdgeIndexFullScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newEdgeIndexFullScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownEdgeIndexFullScanRule::toString() const { - return "PushLimitDownEdgeIndexFullScanRule"; -} - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.h b/src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.h deleted file mode 100644 index a1e20e415ad..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownEdgeIndexFullScanRule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include - -#include "graph/optimizer/OptRule.h" - -namespace nebula { -namespace opt { - -class PushLimitDownEdgeIndexFullScanRule final : public OptRule { - public: - const Pattern &pattern() const override; - - StatusOr transform(OptContext *ctx, - const MatchedResult &matched) const override; - - std::string toString() const override; - - private: - PushLimitDownEdgeIndexFullScanRule(); - - static std::unique_ptr kInstance; -}; - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.cpp deleted file mode 100644 index cb15fcfc80c..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Scan.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::EdgeIndexPrefixScan; -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::QueryContext; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownEdgeIndexPrefixScanRule::kInstance = - std::unique_ptr( - new PushLimitDownEdgeIndexPrefixScanRule()); - -PushLimitDownEdgeIndexPrefixScanRule::PushLimitDownEdgeIndexPrefixScanRule() { - RuleSet::QueryRules().addRule(this); -} - -const Pattern &PushLimitDownEdgeIndexPrefixScanRule::pattern() const { - static Pattern pattern = - Pattern::create(graph::PlanNode::Kind::kLimit, - {Pattern::create(graph::PlanNode::Kind::kEdgeIndexPrefixScan)}); - return pattern; -} - -StatusOr PushLimitDownEdgeIndexPrefixScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newEdgeIndexPrefixScan = static_cast(indexScan->clone()); - newEdgeIndexPrefixScan->setLimit(limitRows); - auto newEdgeIndexPrefixScanGroup = OptGroup::create(octx); - auto newEdgeIndexPrefixScanGroupNode = - newEdgeIndexPrefixScanGroup->makeGroupNode(newEdgeIndexPrefixScan); - - newLimitGroupNode->dependsOn(newEdgeIndexPrefixScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newEdgeIndexPrefixScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownEdgeIndexPrefixScanRule::toString() const { - return "PushLimitDownEdgeIndexPrefixScanRule"; -} - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.h b/src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.h deleted file mode 100644 index cd5956cc346..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownEdgeIndexPrefixScanRule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include - -#include "graph/optimizer/OptRule.h" - -namespace nebula { -namespace opt { - -class PushLimitDownEdgeIndexPrefixScanRule final : public OptRule { - public: - const Pattern &pattern() const override; - - StatusOr transform(OptContext *ctx, - const MatchedResult &matched) const override; - - std::string toString() const override; - - private: - PushLimitDownEdgeIndexPrefixScanRule(); - - static std::unique_ptr kInstance; -}; - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.cpp deleted file mode 100644 index d2443f5d2aa..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Scan.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::EdgeIndexRangeScan; -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::QueryContext; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownEdgeIndexRangeScanRule::kInstance = - std::unique_ptr(new PushLimitDownEdgeIndexRangeScanRule()); - -PushLimitDownEdgeIndexRangeScanRule::PushLimitDownEdgeIndexRangeScanRule() { - RuleSet::QueryRules().addRule(this); -} - -const Pattern &PushLimitDownEdgeIndexRangeScanRule::pattern() const { - static Pattern pattern = Pattern::create( - graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexRangeScan)}); - return pattern; -} - -StatusOr PushLimitDownEdgeIndexRangeScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newEdgeIndexRangeScan = static_cast(indexScan->clone()); - newEdgeIndexRangeScan->setLimit(limitRows); - auto newEdgeIndexRangeScanGroup = OptGroup::create(octx); - auto newEdgeIndexRangeScanGroupNode = - newEdgeIndexRangeScanGroup->makeGroupNode(newEdgeIndexRangeScan); - - newLimitGroupNode->dependsOn(newEdgeIndexRangeScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newEdgeIndexRangeScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownEdgeIndexRangeScanRule::toString() const { - return "PushLimitDownEdgeIndexRangeScanRule"; -} - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.h b/src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.h deleted file mode 100644 index 3bb0db47e25..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownEdgeIndexRangeScanRule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include - -#include "graph/optimizer/OptRule.h" - -namespace nebula { -namespace opt { - -class PushLimitDownEdgeIndexRangeScanRule final : public OptRule { - public: - const Pattern &pattern() const override; - - StatusOr transform(OptContext *ctx, - const MatchedResult &matched) const override; - - std::string toString() const override; - - private: - PushLimitDownEdgeIndexRangeScanRule(); - - static std::unique_ptr kInstance; -}; - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownIndexRule.cpp b/src/graph/optimizer/rule/PushLimitDownIndexRule.cpp new file mode 100644 index 00000000000..0b276c1b7af --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownIndexRule.cpp @@ -0,0 +1,146 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/PushLimitDownIndexRule.h" + +#include "common/expression/FunctionCallExpression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/optimizer/OptRule.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "interface/gen-cpp2/storage_types.h" + +using nebula::graph::IndexScan; +using nebula::graph::Limit; +using nebula::graph::PlanNode; +using nebula::graph::Project; +using nebula::graph::QueryContext; +using nebula::graph::TagIndexFullScan; +using nebula::storage::cpp2::IndexQueryContext; + +namespace nebula { +namespace opt { + +int64_t PushLimitDownIndexRule::limit(const MatchedResult &matched) const { + auto limit = static_cast(matched.node->node()); + int limitRows = limit->offset() + limit->count(); + auto indexScan = static_cast(matched.planNode({0, 0})); + if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { + limitRows = indexScan->limit(); + } + return limitRows; +} +StatusOr> PushLimitDownIndexRule::orderBy( + const MatchedResult &matched) const { + UNUSED(matched); + std::vector orderBys; + return orderBys; +} +OptGroupNode *PushLimitDownIndexRule::topN(OptContext *ctx, const MatchedResult &matched) const { + auto limitGroupNode = matched.node; + const auto limit = static_cast(limitGroupNode->node()); + auto newLimit = static_cast(limit->clone()); + return OptGroupNode::create(ctx, newLimit, limitGroupNode->group()); +} + +std::unique_ptr PushLimitDownTagIndexFullScanRule::kInstance = + std::unique_ptr(new PushLimitDownTagIndexFullScanRule()); +PushLimitDownTagIndexFullScanRule::PushLimitDownTagIndexFullScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushLimitDownTagIndexFullScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kTagIndexFullScan)}); + return pattern; +} +std::string PushLimitDownTagIndexFullScanRule::toString() const { + return "PushLimitDownTagIndexFullScanRule"; +} + +std::unique_ptr PushLimitDownTagIndexPrefixScanRule::kInstance = + std::unique_ptr(new PushLimitDownTagIndexPrefixScanRule()); +PushLimitDownTagIndexPrefixScanRule::PushLimitDownTagIndexPrefixScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushLimitDownTagIndexPrefixScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kTagIndexPrefixScan)}); + return pattern; +} // namespace opt +std::string PushLimitDownTagIndexPrefixScanRule::toString() const { + return "PushLimitDownTagIndexPrefixScanRule"; +} + +std::unique_ptr PushLimitDownTagIndexRangeScanRule::kInstance = + std::unique_ptr(new PushLimitDownTagIndexRangeScanRule()); +PushLimitDownTagIndexRangeScanRule::PushLimitDownTagIndexRangeScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushLimitDownTagIndexRangeScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kTagIndexRangeScan)}); + return pattern; +} // namespace nebula +std::string PushLimitDownTagIndexRangeScanRule::toString() const { + return "PushLimitDownTagIndexRangeScanRule"; +} + +std::unique_ptr PushLimitDownEdgeIndexFullScanRule::kInstance = + std::unique_ptr(new PushLimitDownEdgeIndexFullScanRule()); +PushLimitDownEdgeIndexFullScanRule::PushLimitDownEdgeIndexFullScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushLimitDownEdgeIndexFullScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexFullScan)}); + return pattern; +} +std::string PushLimitDownEdgeIndexFullScanRule::toString() const { + return "PushLimitDownEdgeIndexFullScanRule"; +} + +std::unique_ptr PushLimitDownEdgeIndexPrefixScanRule::kInstance = + std::unique_ptr( + new PushLimitDownEdgeIndexPrefixScanRule()); +PushLimitDownEdgeIndexPrefixScanRule::PushLimitDownEdgeIndexPrefixScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushLimitDownEdgeIndexPrefixScanRule::pattern() const { + static Pattern pattern = + Pattern::create(graph::PlanNode::Kind::kLimit, + {Pattern::create(graph::PlanNode::Kind::kEdgeIndexPrefixScan)}); + return pattern; +} +std::string PushLimitDownEdgeIndexPrefixScanRule::toString() const { + return "PushLimitDownEdgeIndexPrefixScanRule"; +} + +std::unique_ptr PushLimitDownEdgeIndexRangeScanRule::kInstance = + std::unique_ptr(new PushLimitDownEdgeIndexRangeScanRule()); +PushLimitDownEdgeIndexRangeScanRule::PushLimitDownEdgeIndexRangeScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushLimitDownEdgeIndexRangeScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexRangeScan)}); + return pattern; +} +std::string PushLimitDownEdgeIndexRangeScanRule::toString() const { + return "PushLimitDownEdgeIndexRangeScanRule"; +} + +std::unique_ptr PushLimitDownIndexScanRule::kInstance = + std::unique_ptr(new PushLimitDownIndexScanRule()); +PushLimitDownIndexScanRule::PushLimitDownIndexScanRule() { RuleSet::QueryRules().addRule(this); } +const Pattern &PushLimitDownIndexScanRule::pattern() const { + static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kLimit, + {Pattern::create(graph::PlanNode::Kind::kIndexScan)}); + return pattern; +} +std::string PushLimitDownIndexScanRule::toString() const { return "PushLimitDownIndexScanRule"; } +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownIndexRule.h b/src/graph/optimizer/rule/PushLimitDownIndexRule.h new file mode 100644 index 00000000000..84cc32d1ff6 --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitDownIndexRule.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef OPTIMIZER_RULE_PUSHTOPNDOWNTAGFULLINDEXSCAN_H_ +#define OPTIMIZER_RULE_PUSHTOPNDOWNTAGFULLINDEXSCAN_H_ + +#include + +#include "graph/optimizer/OptRule.h" +#include "graph/optimizer/rule/PushLimitSortDownIndexScanRule.h" +#include "graph/planner/plan/Scan.h" + +namespace nebula { +namespace opt { + +class PushLimitDownIndexRule : public PushLimitSortDownIndexScanRule { + protected: + int64_t limit(const MatchedResult &matched) const override; + StatusOr> orderBy( + const MatchedResult &matched) const override; + OptGroupNode *topN(OptContext *ctx, const MatchedResult &matched) const override; +}; + +class PushLimitDownTagIndexFullScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownTagIndexFullScanRule(); + static std::unique_ptr kInstance; +}; + +class PushLimitDownTagIndexPrefixScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownTagIndexPrefixScanRule(); + static std::unique_ptr kInstance; +}; + +class PushLimitDownTagIndexRangeScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownTagIndexRangeScanRule(); + static std::unique_ptr kInstance; +}; + +class PushLimitDownEdgeIndexFullScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownEdgeIndexFullScanRule(); + static std::unique_ptr kInstance; +}; + +class PushLimitDownEdgeIndexPrefixScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownEdgeIndexPrefixScanRule(); + static std::unique_ptr kInstance; +}; + +class PushLimitDownEdgeIndexRangeScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownEdgeIndexRangeScanRule(); + static std::unique_ptr kInstance; +}; + +class PushLimitDownIndexScanRule final : public PushLimitDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushLimitDownIndexScanRule(); + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif // OPTIMIZER_RULE_PUSHTOPNDOWNTAGFULLINDEXSCAN_H_ diff --git a/src/graph/optimizer/rule/PushLimitDownIndexScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownIndexScanRule.cpp deleted file mode 100644 index bee14881cfd..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownIndexScanRule.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownIndexScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Query.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::IndexScan; -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::Project; -using nebula::graph::QueryContext; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownIndexScanRule::kInstance = - std::unique_ptr(new PushLimitDownIndexScanRule()); - -PushLimitDownIndexScanRule::PushLimitDownIndexScanRule() { RuleSet::QueryRules().addRule(this); } - -const Pattern &PushLimitDownIndexScanRule::pattern() const { - static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kLimit, - {Pattern::create(graph::PlanNode::Kind::kIndexScan)}); - return pattern; -} - -StatusOr PushLimitDownIndexScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newIndexScan = static_cast(indexScan->clone()); - newIndexScan->setLimit(limitRows); - auto newIndexScanGroup = OptGroup::create(octx); - auto newIndexScanGroupNode = newIndexScanGroup->makeGroupNode(newIndexScan); - - newLimitGroupNode->dependsOn(newIndexScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newIndexScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownIndexScanRule::toString() const { return "PushLimitDownIndexScanRule"; } - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownProjectRule.cpp b/src/graph/optimizer/rule/PushLimitDownProjectRule.cpp index 21f3d65420b..d8f31080382 100644 --- a/src/graph/optimizer/rule/PushLimitDownProjectRule.cpp +++ b/src/graph/optimizer/rule/PushLimitDownProjectRule.cpp @@ -46,6 +46,8 @@ StatusOr PushLimitDownProjectRule::transform( const auto limit = static_cast(limitGroupNode->node()); const auto proj = static_cast(projGroupNode->node()); + auto projColNames = proj->colNames(); + auto newLimit = static_cast(limit->clone()); auto newLimitGroup = OptGroup::create(octx); auto newLimitGroupNode = newLimitGroup->makeGroupNode(newLimit); diff --git a/src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.cpp deleted file mode 100644 index 4352c2db9b3..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Scan.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::QueryContext; -using nebula::graph::TagIndexFullScan; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownTagIndexFullScanRule::kInstance = - std::unique_ptr(new PushLimitDownTagIndexFullScanRule()); - -PushLimitDownTagIndexFullScanRule::PushLimitDownTagIndexFullScanRule() { - RuleSet::QueryRules().addRule(this); -} - -const Pattern &PushLimitDownTagIndexFullScanRule::pattern() const { - static Pattern pattern = Pattern::create( - graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kTagIndexFullScan)}); - return pattern; -} - -StatusOr PushLimitDownTagIndexFullScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newTagIndexFullScan = static_cast(indexScan->clone()); - newTagIndexFullScan->setLimit(limitRows); - auto newTagIndexFullScanGroup = OptGroup::create(octx); - auto newTagIndexFullScanGroupNode = newTagIndexFullScanGroup->makeGroupNode(newTagIndexFullScan); - - newLimitGroupNode->dependsOn(newTagIndexFullScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newTagIndexFullScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownTagIndexFullScanRule::toString() const { - return "PushLimitDownTagIndexFullScanRule"; -} - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.h b/src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.h deleted file mode 100644 index 32bd4cbe108..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownTagIndexFullScanRule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include - -#include "graph/optimizer/OptRule.h" - -namespace nebula { -namespace opt { - -class PushLimitDownTagIndexFullScanRule final : public OptRule { - public: - const Pattern &pattern() const override; - - StatusOr transform(OptContext *ctx, - const MatchedResult &matched) const override; - - std::string toString() const override; - - private: - PushLimitDownTagIndexFullScanRule(); - - static std::unique_ptr kInstance; -}; - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.cpp deleted file mode 100644 index 358d79f17b0..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Scan.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::QueryContext; -using nebula::graph::TagIndexPrefixScan; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownTagIndexPrefixScanRule::kInstance = - std::unique_ptr(new PushLimitDownTagIndexPrefixScanRule()); - -PushLimitDownTagIndexPrefixScanRule::PushLimitDownTagIndexPrefixScanRule() { - RuleSet::QueryRules().addRule(this); -} - -const Pattern &PushLimitDownTagIndexPrefixScanRule::pattern() const { - static Pattern pattern = Pattern::create( - graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kTagIndexPrefixScan)}); - return pattern; -} - -StatusOr PushLimitDownTagIndexPrefixScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newTagIndexPrefixScan = static_cast(indexScan->clone()); - newTagIndexPrefixScan->setLimit(limitRows); - auto newTagIndexPrefixScanGroup = OptGroup::create(octx); - auto newTagIndexPrefixScanGroupNode = - newTagIndexPrefixScanGroup->makeGroupNode(newTagIndexPrefixScan); - - newLimitGroupNode->dependsOn(newTagIndexPrefixScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newTagIndexPrefixScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownTagIndexPrefixScanRule::toString() const { - return "PushLimitDownTagIndexPrefixScanRule"; -} - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.h b/src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.h deleted file mode 100644 index a12f58b71c1..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownTagIndexPrefixScanRule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include - -#include "graph/optimizer/OptRule.h" - -namespace nebula { -namespace opt { - -class PushLimitDownTagIndexPrefixScanRule final : public OptRule { - public: - const Pattern &pattern() const override; - - StatusOr transform(OptContext *ctx, - const MatchedResult &matched) const override; - - std::string toString() const override; - - private: - PushLimitDownTagIndexPrefixScanRule(); - - static std::unique_ptr kInstance; -}; - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.cpp b/src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.cpp deleted file mode 100644 index c46edeba4bc..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#include "graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.h" - -#include "common/expression/BinaryExpression.h" -#include "common/expression/ConstantExpression.h" -#include "common/expression/Expression.h" -#include "common/expression/FunctionCallExpression.h" -#include "common/expression/LogicalExpression.h" -#include "common/expression/UnaryExpression.h" -#include "graph/optimizer/OptContext.h" -#include "graph/optimizer/OptGroup.h" -#include "graph/planner/plan/PlanNode.h" -#include "graph/planner/plan/Scan.h" -#include "graph/visitor/ExtractFilterExprVisitor.h" - -using nebula::graph::Limit; -using nebula::graph::PlanNode; -using nebula::graph::QueryContext; -using nebula::graph::TagIndexRangeScan; - -namespace nebula { -namespace opt { - -std::unique_ptr PushLimitDownTagIndexRangeScanRule::kInstance = - std::unique_ptr(new PushLimitDownTagIndexRangeScanRule()); - -PushLimitDownTagIndexRangeScanRule::PushLimitDownTagIndexRangeScanRule() { - RuleSet::QueryRules().addRule(this); -} - -const Pattern &PushLimitDownTagIndexRangeScanRule::pattern() const { - static Pattern pattern = Pattern::create( - graph::PlanNode::Kind::kLimit, {Pattern::create(graph::PlanNode::Kind::kTagIndexRangeScan)}); - return pattern; -} - -StatusOr PushLimitDownTagIndexRangeScanRule::transform( - OptContext *octx, const MatchedResult &matched) const { - auto limitGroupNode = matched.node; - auto indexScanGroupNode = matched.dependencies.front().node; - - const auto limit = static_cast(limitGroupNode->node()); - const auto indexScan = static_cast(indexScanGroupNode->node()); - - int64_t limitRows = limit->offset() + limit->count(); - if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { - return TransformResult::noTransform(); - } - - auto newLimit = static_cast(limit->clone()); - auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group()); - - auto newTagIndexRangeScan = static_cast(indexScan->clone()); - newTagIndexRangeScan->setLimit(limitRows); - auto newTagIndexRangeScanGroup = OptGroup::create(octx); - auto newTagIndexRangeScanGroupNode = - newTagIndexRangeScanGroup->makeGroupNode(newTagIndexRangeScan); - - newLimitGroupNode->dependsOn(newTagIndexRangeScanGroup); - for (auto dep : indexScanGroupNode->dependencies()) { - newTagIndexRangeScanGroupNode->dependsOn(dep); - } - - TransformResult result; - result.eraseAll = true; - result.newGroupNodes.emplace_back(newLimitGroupNode); - return result; -} - -std::string PushLimitDownTagIndexRangeScanRule::toString() const { - return "PushLimitDownTagIndexRangeScanRule"; -} - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.h b/src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.h deleted file mode 100644 index 3f66b998344..00000000000 --- a/src/graph/optimizer/rule/PushLimitDownTagIndexRangeScanRule.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2021 vesoft inc. All rights reserved. - * - * This source code is licensed under Apache 2.0 License, - * attached with Common Clause Condition 1.0, found in the LICENSES directory. - */ - -#pragma once - -#include - -#include "graph/optimizer/OptRule.h" - -namespace nebula { -namespace opt { - -class PushLimitDownTagIndexRangeScanRule final : public OptRule { - public: - const Pattern &pattern() const override; - - StatusOr transform(OptContext *ctx, - const MatchedResult &matched) const override; - - std::string toString() const override; - - private: - PushLimitDownTagIndexRangeScanRule(); - - static std::unique_ptr kInstance; -}; - -} // namespace opt -} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.cpp b/src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.cpp new file mode 100644 index 00000000000..63692e3f59f --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.cpp @@ -0,0 +1,58 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/PushLimitSortDownIndexScanRule.h" + +#include "common/expression/FunctionCallExpression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/optimizer/OptRule.h" +#include "graph/planner/plan/Query.h" +#include "interface/gen-cpp2/storage_types.h" + +using nebula::graph::IndexScan; +using nebula::graph::PlanNode; +using nebula::graph::Project; +using nebula::graph::QueryContext; +using nebula::graph::TagIndexFullScan; +using nebula::graph::TopN; +using nebula::storage::cpp2::IndexQueryContext; + +namespace nebula { +namespace opt { + +StatusOr PushLimitSortDownIndexScanRule::transform( + OptContext *ctx, const MatchedResult &matched) const { + int64_t limitRows = limit(matched); + auto ret = orderBy(matched); + if (!ret.ok()) { + return TransformResult::noTransform(); + } + std::vector orderBys = std::move(ret.value()); + + auto newTopNGroupNode = topN(ctx, matched); + + auto indexScanGroupNode = matched.dependencies.front().node; + const auto indexScan = static_cast(indexScanGroupNode->node()); + auto newIndexScan = static_cast(indexScan->clone()); + newIndexScan->setLimit(limitRows); + newIndexScan->setOrderBy(std::move(orderBys)); + auto newIndexGroup = OptGroup::create(ctx); + auto newIndexGroupNode = newIndexGroup->makeGroupNode(newIndexScan); + + newTopNGroupNode->dependsOn(newIndexGroup); + for (auto dep : indexScanGroupNode->dependencies()) { + newIndexGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newTopNGroupNode); + return result; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.h b/src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.h new file mode 100644 index 00000000000..07f6e6f9b4b --- /dev/null +++ b/src/graph/optimizer/rule/PushLimitSortDownIndexScanRule.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef OPTIMIZER_RULE_PUSHLIMITSORTDOWNINDEXSCAN_H_ +#define OPTIMIZER_RULE_PUSHLIMITSORTDOWNINDEXSCAN_H_ + +#include + +#include "graph/optimizer/OptRule.h" +#include "graph/planner/plan/Scan.h" + +namespace nebula { +namespace opt { + +class PushLimitSortDownIndexScanRule : public OptRule { + public: + StatusOr transform(OptContext *ctx, + const MatchedResult &matched) const override; + + protected: + virtual int64_t limit(const MatchedResult &matched) const = 0; + virtual StatusOr> orderBy( + const MatchedResult &matched) const = 0; + virtual OptGroupNode *topN(OptContext *ctx, const MatchedResult &matched) const = 0; +}; + +} // namespace opt +} // namespace nebula + +#endif // OPTIMIZER_RULE_PUSHLIMITSORTDOWNINDEXSCAN_H_ diff --git a/src/graph/optimizer/rule/PushTopNDownIndexRule.cpp b/src/graph/optimizer/rule/PushTopNDownIndexRule.cpp new file mode 100644 index 00000000000..acb10d48e17 --- /dev/null +++ b/src/graph/optimizer/rule/PushTopNDownIndexRule.cpp @@ -0,0 +1,176 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/PushTopNDownIndexRule.h" + +#include "common/expression/FunctionCallExpression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/optimizer/OptRule.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "interface/gen-cpp2/storage_types.h" + +using nebula::graph::IndexScan; +using nebula::graph::PlanNode; +using nebula::graph::Project; +using nebula::graph::QueryContext; +using nebula::graph::TagIndexFullScan; +using nebula::graph::TopN; +using nebula::storage::cpp2::IndexQueryContext; + +namespace nebula { +namespace opt { + +int64_t PushTopNDownIndexRule::limit(const MatchedResult &matched) const { + auto topN = static_cast(matched.node->node()); + int limitRows = topN->offset() + topN->count(); + auto indexScan = static_cast(matched.planNode({0, 0})); + if (indexScan->limit() >= 0 && limitRows >= indexScan->limit()) { + limitRows = indexScan->limit(); + } + return limitRows; +} +StatusOr> PushTopNDownIndexRule::orderBy( + const MatchedResult &matched) const { + auto indexScan = static_cast(matched.planNode({0, 0})); + if (!indexScan->orderBy().empty()) { + return Status::Error(); + } + auto indexReturnColNames = indexScan->returnColumns(); + + auto topN = static_cast(matched.node->node()); + auto &factors = topN->factors(); + auto factorColNames = topN->factorColNames(); + if (factorColNames.empty()) { + return Status::Error(); + } + std::vector orderBys; + orderBys.reserve(factors.size()); + const auto &outPutColsToReturnColsMap = indexScan->outPutColsToReturnCols(); + for (auto factor : factors) { + auto colName = factorColNames[factor.first]; + auto found = outPutColsToReturnColsMap.find(colName); + if (found == outPutColsToReturnColsMap.end()) { + return Status::Error(); + } + colName = found->second; + auto eq = [&](const std::string &name) { return name == colName; }; + auto iter = std::find_if(indexReturnColNames.cbegin(), indexReturnColNames.cend(), eq); + size_t colIdx = std::distance(indexReturnColNames.cbegin(), iter); + storage::cpp2::OrderBy orderBy; + orderBy.set_pos(colIdx); + orderBy.set_prop(""); + orderBy.set_direction(factor.second == OrderFactor::OrderType::ASCEND + ? storage::cpp2::OrderDirection::ASCENDING + : storage::cpp2::OrderDirection::DESCENDING); + orderBys.emplace_back(orderBy); + } + return orderBys; +} +OptGroupNode *PushTopNDownIndexRule::topN(OptContext *ctx, const MatchedResult &matched) const { + auto topNGroupNode = matched.node; + const auto topN = static_cast(topNGroupNode->node()); + auto newTopN = static_cast(topN->clone()); + return OptGroupNode::create(ctx, newTopN, topNGroupNode->group()); +} + +std::unique_ptr PushTopNDownTagIndexFullScanRule::kInstance = + std::unique_ptr(new PushTopNDownTagIndexFullScanRule()); +PushTopNDownTagIndexFullScanRule::PushTopNDownTagIndexFullScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushTopNDownTagIndexFullScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kTopN, {Pattern::create(graph::PlanNode::Kind::kTagIndexFullScan)}); + return pattern; +} +std::string PushTopNDownTagIndexFullScanRule::toString() const { + return "PushTopNDownTagIndexFullScanRule"; +} + +std::unique_ptr PushTopNDownTagIndexPrefixScanRule::kInstance = + std::unique_ptr(new PushTopNDownTagIndexPrefixScanRule()); +PushTopNDownTagIndexPrefixScanRule::PushTopNDownTagIndexPrefixScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushTopNDownTagIndexPrefixScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kTopN, {Pattern::create(graph::PlanNode::Kind::kTagIndexPrefixScan)}); + return pattern; +} +std::string PushTopNDownTagIndexPrefixScanRule::toString() const { + return "PushTopNDownTagIndexPrefixScanRule"; +} + +std::unique_ptr PushTopNDownTagIndexRangeScanRule::kInstance = + std::unique_ptr(new PushTopNDownTagIndexRangeScanRule()); +PushTopNDownTagIndexRangeScanRule::PushTopNDownTagIndexRangeScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushTopNDownTagIndexRangeScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kTopN, {Pattern::create(graph::PlanNode::Kind::kTagIndexRangeScan)}); + return pattern; +} +std::string PushTopNDownTagIndexRangeScanRule::toString() const { + return "PushTopNDownTagIndexRangeScanRule"; +} + +std::unique_ptr PushTopNDownEdgeIndexFullScanRule::kInstance = + std::unique_ptr(new PushTopNDownEdgeIndexFullScanRule()); +PushTopNDownEdgeIndexFullScanRule::PushTopNDownEdgeIndexFullScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushTopNDownEdgeIndexFullScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kTopN, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexFullScan)}); + return pattern; +} +std::string PushTopNDownEdgeIndexFullScanRule::toString() const { + return "PushTopNDownEdgeIndexFullScanRule"; +} + +std::unique_ptr PushTopNDownEdgeIndexPrefixScanRule::kInstance = + std::unique_ptr(new PushTopNDownEdgeIndexPrefixScanRule()); +PushTopNDownEdgeIndexPrefixScanRule::PushTopNDownEdgeIndexPrefixScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushTopNDownEdgeIndexPrefixScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kTopN, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexPrefixScan)}); + return pattern; +} +std::string PushTopNDownEdgeIndexPrefixScanRule::toString() const { + return "PushTopNDownEdgeIndexPrefixScanRule"; +} + +std::unique_ptr PushTopNDownEdgeIndexRangeScanRule::kInstance = + std::unique_ptr(new PushTopNDownEdgeIndexRangeScanRule()); +PushTopNDownEdgeIndexRangeScanRule::PushTopNDownEdgeIndexRangeScanRule() { + RuleSet::QueryRules().addRule(this); +} +const Pattern &PushTopNDownEdgeIndexRangeScanRule::pattern() const { + static Pattern pattern = Pattern::create( + graph::PlanNode::Kind::kTopN, {Pattern::create(graph::PlanNode::Kind::kEdgeIndexRangeScan)}); + return pattern; +} +std::string PushTopNDownEdgeIndexRangeScanRule::toString() const { + return "PushTopNDownEdgeIndexRangeScanRule"; +} + +std::unique_ptr PushTopNDownIndexScanRule::kInstance = + std::unique_ptr(new PushTopNDownIndexScanRule()); +PushTopNDownIndexScanRule::PushTopNDownIndexScanRule() { RuleSet::QueryRules().addRule(this); } +const Pattern &PushTopNDownIndexScanRule::pattern() const { + static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kTopN, + {Pattern::create(graph::PlanNode::Kind::kIndexScan)}); + return pattern; +} +std::string PushTopNDownIndexScanRule::toString() const { return "PushTopNDownIndexScanRule"; } + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushTopNDownIndexRule.h b/src/graph/optimizer/rule/PushTopNDownIndexRule.h new file mode 100644 index 00000000000..a28be247c7f --- /dev/null +++ b/src/graph/optimizer/rule/PushTopNDownIndexRule.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef OPTIMIZER_RULE_PUSHTOPNDOWNTAGFULLINDEXSCAN_H_ +#define OPTIMIZER_RULE_PUSHTOPNDOWNTAGFULLINDEXSCAN_H_ + +#include + +#include "graph/optimizer/OptRule.h" +#include "graph/optimizer/rule/PushLimitSortDownIndexScanRule.h" +#include "graph/planner/plan/Scan.h" + +namespace nebula { +namespace opt { + +class PushTopNDownIndexRule : public PushLimitSortDownIndexScanRule { + protected: + int64_t limit(const MatchedResult &matched) const override; + StatusOr> orderBy( + const MatchedResult &matched) const override; + OptGroupNode *topN(OptContext *ctx, const MatchedResult &matched) const override; +}; + +class PushTopNDownTagIndexFullScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownTagIndexFullScanRule(); + static std::unique_ptr kInstance; +}; + +class PushTopNDownTagIndexPrefixScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownTagIndexPrefixScanRule(); + static std::unique_ptr kInstance; +}; + +class PushTopNDownTagIndexRangeScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownTagIndexRangeScanRule(); + static std::unique_ptr kInstance; +}; + +class PushTopNDownEdgeIndexFullScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownEdgeIndexFullScanRule(); + static std::unique_ptr kInstance; +}; + +class PushTopNDownEdgeIndexPrefixScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownEdgeIndexPrefixScanRule(); + static std::unique_ptr kInstance; +}; + +class PushTopNDownEdgeIndexRangeScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownEdgeIndexRangeScanRule(); + static std::unique_ptr kInstance; +}; + +class PushTopNDownIndexScanRule final : public PushTopNDownIndexRule { + public: + const Pattern &pattern() const override; + std::string toString() const override; + + private: + PushTopNDownIndexScanRule(); + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula + +#endif // OPTIMIZER_RULE_PUSHTOPNDOWNTAGFULLINDEXSCAN_H_ diff --git a/src/graph/optimizer/rule/PushTopNDownProjectRule.cpp b/src/graph/optimizer/rule/PushTopNDownProjectRule.cpp new file mode 100644 index 00000000000..4dc3341a4cc --- /dev/null +++ b/src/graph/optimizer/rule/PushTopNDownProjectRule.cpp @@ -0,0 +1,73 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#include "graph/optimizer/rule/PushTopNDownProjectRule.h" + +#include "common/expression/FunctionCallExpression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" + +using nebula::graph::PlanNode; +using nebula::graph::Project; +using nebula::graph::QueryContext; +using nebula::graph::TopN; + +namespace nebula { +namespace opt { + +// transform TopN->Project to Project->TopN +std::unique_ptr PushTopNDownProjectRule::kInstance = + std::unique_ptr(new PushTopNDownProjectRule()); + +PushTopNDownProjectRule::PushTopNDownProjectRule() { RuleSet::QueryRules().addRule(this); } + +const Pattern &PushTopNDownProjectRule::pattern() const { + static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kTopN, + {Pattern::create(graph::PlanNode::Kind::kProject)}); + return pattern; +} + +StatusOr PushTopNDownProjectRule::transform( + OptContext *octx, const MatchedResult &matched) const { + auto topNGroupNode = matched.node; + auto projGroupNode = matched.dependencies.front().node; + + const auto topN = static_cast(topNGroupNode->node()); + const auto proj = static_cast(projGroupNode->node()); + + auto projColNames = proj->colNames(); + + auto newTopN = static_cast(topN->clone()); + auto newTopNGroup = OptGroup::create(octx); + auto newTopNGroupNode = newTopNGroup->makeGroupNode(newTopN); + newTopN->setOutputVar(proj->outputVar()); + newTopN->setInputVar(proj->inputVar()); + newTopN->setColNames(newTopN->inputVars()[0]->colNames); + newTopN->setFactorColNames(projColNames); + + auto newProj = static_cast(proj->clone()); + auto newProjGroupNode = OptGroupNode::create(octx, newProj, topNGroupNode->group()); + newProj->setOutputVar(topN->outputVar()); + newProj->setInputVar(newTopN->outputVar()); + newProj->setColNames(projColNames); + + newProjGroupNode->dependsOn(const_cast(newTopNGroupNode->group())); + for (auto dep : projGroupNode->dependencies()) { + newTopNGroupNode->dependsOn(dep); + } + + TransformResult result; + result.eraseAll = true; + result.newGroupNodes.emplace_back(newProjGroupNode); + return result; +} + +std::string PushTopNDownProjectRule::toString() const { return "PushTopNDownProjectRule"; } + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/PushLimitDownIndexScanRule.h b/src/graph/optimizer/rule/PushTopNDownProjectRule.h similarity index 88% rename from src/graph/optimizer/rule/PushLimitDownIndexScanRule.h rename to src/graph/optimizer/rule/PushTopNDownProjectRule.h index 0b3aa5d4078..d33bf2a5048 100644 --- a/src/graph/optimizer/rule/PushLimitDownIndexScanRule.h +++ b/src/graph/optimizer/rule/PushTopNDownProjectRule.h @@ -13,7 +13,7 @@ namespace nebula { namespace opt { -class PushLimitDownIndexScanRule final : public OptRule { +class PushTopNDownProjectRule final : public OptRule { public: const Pattern &pattern() const override; @@ -23,7 +23,7 @@ class PushLimitDownIndexScanRule final : public OptRule { std::string toString() const override; private: - PushLimitDownIndexScanRule(); + PushTopNDownProjectRule(); static std::unique_ptr kInstance; }; diff --git a/src/graph/optimizer/rule/TopNRule.cpp b/src/graph/optimizer/rule/TopNRule.cpp index 80a7faebdf7..155fb29310e 100644 --- a/src/graph/optimizer/rule/TopNRule.cpp +++ b/src/graph/optimizer/rule/TopNRule.cpp @@ -44,12 +44,6 @@ StatusOr TopNRule::transform(OptContext *ctx, auto limit = static_cast(limitGroupNode->node()); auto sort = static_cast(sortGroupNode->node()); - // Currently, we cannot know the total amount of input data, - // so only apply topn rule when offset of limit is 0 - if (limit->offset() != 0) { - return TransformResult::noTransform(); - } - auto qctx = ctx->qctx(); auto topn = TopN::make(qctx, nullptr, sort->factors(), limit->offset(), limit->count()); topn->setOutputVar(limit->outputVar()); diff --git a/src/graph/planner/ngql/LookupPlanner.cpp b/src/graph/planner/ngql/LookupPlanner.cpp index 3a51e6eb570..7dde631fcbc 100644 --- a/src/graph/planner/ngql/LookupPlanner.cpp +++ b/src/graph/planner/ngql/LookupPlanner.cpp @@ -40,6 +40,7 @@ StatusOr LookupPlanner::transform(AstContext* astCtx) { lookupCtx->isEmptyResultSet); plan.tail = edgeIndexFullScan; plan.root = edgeIndexFullScan; + edgeIndexFullScan->setOutPutColsToReturnCols(lookupCtx->idxOutColsToReturnColsMap_); } else { auto* tagIndexFullScan = TagIndexFullScan::make(qctx, nullptr, @@ -51,6 +52,7 @@ StatusOr LookupPlanner::transform(AstContext* astCtx) { lookupCtx->isEmptyResultSet); plan.tail = tagIndexFullScan; plan.root = tagIndexFullScan; + tagIndexFullScan->setOutPutColsToReturnCols(lookupCtx->idxOutColsToReturnColsMap_); } plan.tail->setColNames(lookupCtx->idxColNames); diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp index 07af8ff0731..597cc70ebfd 100644 --- a/src/graph/planner/plan/Query.cpp +++ b/src/graph/planner/plan/Query.cpp @@ -184,6 +184,7 @@ void IndexScan::cloneMembers(const IndexScan& g) { contexts_ = g.contexts_; returnCols_ = g.returnCols_; + outPutColsToReturnColsMap_ = g.outPutColsToReturnColsMap_; isEdge_ = g.isEdge(); schemaId_ = g.schemaId(); isEmptyResultSet_ = g.isEmptyResultSet(); @@ -361,6 +362,7 @@ void TopN::cloneMembers(const TopN& l) { factors.emplace_back(factor); } factors_ = std::move(factors); + factorColNames_ = l.factorColNames_; offset_ = l.offset_; count_ = l.count_; } diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h index e94344c3f5e..cdd5840b018 100644 --- a/src/graph/planner/plan/Query.h +++ b/src/graph/planner/plan/Query.h @@ -412,6 +412,10 @@ class IndexScan : public Explore { const std::vector& returnColumns() const { return returnCols_; } + const std::unordered_map& outPutColsToReturnCols() const { + return outPutColsToReturnColsMap_; + } + bool isEdge() const { return isEdge_; } int32_t schemaId() const { return schemaId_; } @@ -428,6 +432,11 @@ class IndexScan : public Explore { void setReturnCols(std::vector cols) { returnCols_ = std::move(cols); } + void setOutPutColsToReturnCols( + std::unordered_map outPutColsToReturnColsMap) { + outPutColsToReturnColsMap_ = std::move(outPutColsToReturnColsMap); + } + void setIsEdge(bool isEdge) { isEdge_ = isEdge; } PlanNode* clone() const override; @@ -460,6 +469,7 @@ class IndexScan : public Explore { private: std::vector contexts_; std::vector returnCols_; + std::unordered_map outPutColsToReturnColsMap_; bool isEdge_; int32_t schemaId_; @@ -736,17 +746,23 @@ class TopN final : public SingleInputNode { static TopN* make(QueryContext* qctx, PlanNode* input, std::vector> factors = {}, - int64_t offset = -1, - int64_t count = -1) { + int64_t offset = 0, + int64_t count = 0) { return qctx->objPool()->add(new TopN(qctx, input, std::move(factors), offset, count)); } const std::vector>& factors() const { return factors_; } + const std::vector& factorColNames() const { return factorColNames_; } + int64_t offset() const { return offset_; } int64_t count() const { return count_; } + void setFactorColNames(const std::vector& factorColNames) { + factorColNames_ = factorColNames; + } + PlanNode* clone() const override; std::unique_ptr explain() const override; @@ -782,6 +798,7 @@ class TopN final : public SingleInputNode { private: std::vector> factors_; + std::vector factorColNames_; int64_t offset_{-1}; int64_t count_{-1}; }; diff --git a/src/graph/util/ToJson.cpp b/src/graph/util/ToJson.cpp index eee58d54980..3ce19a15783 100644 --- a/src/graph/util/ToJson.cpp +++ b/src/graph/util/ToJson.cpp @@ -171,6 +171,9 @@ folly::dynamic toJson(const storage::cpp2::OrderBy &orderBy) { if (orderBy.prop_ref().is_set()) { obj.insert("prop", *orderBy.prop_ref()); } + if (orderBy.pos_ref().is_set()) { + obj.insert("pos", toJson(*orderBy.pos_ref())); + } return obj; } diff --git a/src/graph/validator/LookupValidator.cpp b/src/graph/validator/LookupValidator.cpp index 615354ac0b3..9ea8d6a9ed8 100644 --- a/src/graph/validator/LookupValidator.cpp +++ b/src/graph/validator/LookupValidator.cpp @@ -98,6 +98,7 @@ void LookupValidator::extractExprProps() { lookupCtx_->idxColNames = std::move(idxColNames); } lookupCtx_->idxReturnCols = std::move(idxReturnCols_); + lookupCtx_->idxOutColsToReturnColsMap_ = std::move(idxOutColsToReturnColsMap_); } Status LookupValidator::validateYieldEdge() { @@ -126,6 +127,10 @@ Status LookupValidator::validateYieldEdge() { outputs_.emplace_back(col->name(), typeStatus.value()); yieldExpr->addColumn(col->clone().release()); NG_RETURN_IF_ERROR(deduceProps(colExpr, exprProps_)); + if (col->expr()->kind() == Expression::Kind::kEdgeProperty) { + const auto& propName = static_cast(col->expr())->prop(); + idxOutColsToReturnColsMap_.emplace(col->name(), propName); + } } return Status::OK(); } @@ -156,6 +161,10 @@ Status LookupValidator::validateYieldTag() { outputs_.emplace_back(col->name(), typeStatus.value()); yieldExpr->addColumn(col->clone().release()); NG_RETURN_IF_ERROR(deduceProps(colExpr, exprProps_)); + if (col->expr()->kind() == Expression::Kind::kTagProperty) { + const auto& propName = static_cast(col->expr())->prop(); + idxOutColsToReturnColsMap_.emplace(col->name(), propName); + } } return Status::OK(); } @@ -175,9 +184,13 @@ Status LookupValidator::validateYield() { newCols->addColumn(new YieldColumn(ColumnExpression::make(pool, 0), kSrcVID)); newCols->addColumn(new YieldColumn(ColumnExpression::make(pool, 1), kDstVID)); newCols->addColumn(new YieldColumn(ColumnExpression::make(pool, 2), kRanking)); + idxOutColsToReturnColsMap_.emplace(kSrcVID, kSrc); + idxOutColsToReturnColsMap_.emplace(kDstVID, kDst); + idxOutColsToReturnColsMap_.emplace(kRanking, kRank); } else { idxReturnCols_.emplace_back(kVid); outputs_.emplace_back(kVertexID, vidType_); + idxOutColsToReturnColsMap_.emplace(kVertexID, kVid); newCols->addColumn(new YieldColumn(ColumnExpression::make(pool, 0), kVertexID)); } diff --git a/src/graph/validator/LookupValidator.h b/src/graph/validator/LookupValidator.h index 9d0178ad35c..ecdcad369a0 100644 --- a/src/graph/validator/LookupValidator.h +++ b/src/graph/validator/LookupValidator.h @@ -61,6 +61,7 @@ class LookupValidator final : public Validator { std::vector tsClients_; ExpressionProps exprProps_; std::vector idxReturnCols_; + std::unordered_map idxOutColsToReturnColsMap_; }; } // namespace graph diff --git a/src/interface/storage.thrift b/src/interface/storage.thrift index 2c61e5f3648..737c5c96463 100644 --- a/src/interface/storage.thrift +++ b/src/interface/storage.thrift @@ -110,6 +110,7 @@ struct OrderBy { // An expression which result will be used to sort 1: binary prop, 2: OrderDirection direction, + 3: i32 pos, } @@ -526,6 +527,8 @@ struct IndexSpec { // In order to union multiple indices, multiple index hints are allowed 1: required list contexts, 2: common.SchemaID schema_id, + 3: optional list order_by, + 4: optional i64 limit, } diff --git a/src/storage/exec/TopKNode.h b/src/storage/exec/TopKNode.h new file mode 100644 index 00000000000..92162e0b517 --- /dev/null +++ b/src/storage/exec/TopKNode.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2021 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License, + * attached with Common Clause Condition 1.0, found in the LICENSES directory. + */ + +#ifndef STORAGE_EXEC_TOPKNODE_H_ +#define STORAGE_EXEC_TOPKNODE_H_ + +#include "common/base/Base.h" +#include "common/utils/TopKHeap.h" +#include "storage/exec/RelNode.h" + +namespace nebula { +namespace storage { + +// TopKNode will return a DataSet with fixed size +template +class TopKNode final : public RelNode { + public: + using RelNode::doExecute; + + TopKNode(nebula::DataSet* resultSet, std::vector orderBy, int limit) + : resultSet_(resultSet), orderBy_(orderBy), limit_(limit) { + RelNode::name_ = "TopKNode"; + } + + nebula::cpp2::ErrorCode doExecute(PartitionID partId) override { + auto ret = RelNode::doExecute(partId); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } + if (limit_ > 0) { + resultSet_->rows = topK(resultSet_->rows, orderBy_, limit_); + } + return nebula::cpp2::ErrorCode::SUCCEEDED; + } + + static std::vector topK(std::vector& rows, std::vector orderBy, int k) { + auto comparator = [orderBy](const Row& lhs, const Row& rhs) { + for (auto& item : orderBy) { + auto index = item.get_pos(); + auto orderType = item.get_direction(); + if (lhs[index] == rhs[index]) { + continue; + } + if (orderType == cpp2::OrderDirection::ASCENDING) { + return lhs[index] < rhs[index]; + } else if (orderType == cpp2::OrderDirection::DESCENDING) { + return lhs[index] > rhs[index]; + } + } + return false; + }; + TopKHeap topKHeap(k, std::move(comparator)); + for (auto row : rows) { + topKHeap.push(row); + } + return topKHeap.moveTopK(); + } + + private: + nebula::DataSet* resultSet_; + std::vector orderBy_{}; + int limit_{-1}; +}; + +} // namespace storage +} // namespace nebula + +#endif // STORAGE_EXEC_TOPKNODE_H_ diff --git a/src/storage/index/LookupBaseProcessor-inl.h b/src/storage/index/LookupBaseProcessor-inl.h index 6a280f078bb..73ab6193721 100644 --- a/src/storage/index/LookupBaseProcessor-inl.h +++ b/src/storage/index/LookupBaseProcessor-inl.h @@ -64,6 +64,12 @@ nebula::cpp2::ErrorCode LookupBaseProcessor::requestCheck( return nebula::cpp2::ErrorCode::E_INVALID_OPERATION; } indexContexts_ = indices.get_contexts(); + if (indices.order_by_ref().has_value()) { + orderBy_ = *indices.get_order_by(); + } + if (indices.limit_ref().has_value()) { + limit_ = *indices.get_limit(); + } // setup yield columns. if (req.return_columns_ref().has_value()) { @@ -77,15 +83,6 @@ nebula::cpp2::ErrorCode LookupBaseProcessor::requestCheck( } } - // limit - if (req.limit_ref().has_value()) { - if (*req.limit_ref() < 0) { - LOG(ERROR) << "Incorrect parameter : LIMIT = " << *req.limit_ref(); - return nebula::cpp2::ErrorCode::E_INVALID_PARM; - } - limit_ = *req.limit_ref(); - } - return nebula::cpp2::ErrorCode::SUCCEEDED; } @@ -171,6 +168,11 @@ StatusOr> LookupBaseProcessor::buildPlan( // TODO(sky) : Limit is not supported yet for de-dup node. // Related to paging scan, the de-dup execution plan needs to be refactored auto deDup = std::make_unique>(result, deDupColPos_); + std::unique_ptr> topK = nullptr; + if (!orderBy_.empty()) { + topK = std::make_unique>(result, orderBy_, limit_); + limit_ = -1; + } int32_t filterId = 0; std::unique_ptr> out; auto pool = &planContext_->objPool_; @@ -262,9 +264,17 @@ StatusOr> LookupBaseProcessor::buildPlan( if (out == nullptr) { return Status::Error("Index scan plan error"); } - deDup->addDependency(out.get()); + if (topK != nullptr) { + topK->addDependency(out.get()); + } else { + deDup->addDependency(out.get()); + } plan.addNode(std::move(out)); } + if (topK != nullptr) { + deDup->addDependency(topK.get()); + plan.addNode(std::move(topK)); + } plan.addNode(std::move(deDup)); return plan; } diff --git a/src/storage/index/LookupBaseProcessor.h b/src/storage/index/LookupBaseProcessor.h index 1815e8a72fa..c005cf9da9d 100644 --- a/src/storage/index/LookupBaseProcessor.h +++ b/src/storage/index/LookupBaseProcessor.h @@ -17,6 +17,7 @@ #include "storage/exec/IndexScanNode.h" #include "storage/exec/IndexVertexNode.h" #include "storage/exec/StoragePlan.h" +#include "storage/exec/TopKNode.h" namespace nebula { namespace storage { @@ -83,7 +84,8 @@ class LookupBaseProcessor : public BaseProcessor { // Save schemas when column is out of index, need to read from data std::vector> schemas_; std::vector deDupColPos_; - int64_t limit_ = -1; + int limit_ = -1; + std::vector orderBy_{}; }; } // namespace storage diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index d25be24594a..6631f275e32 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -103,6 +103,9 @@ void LookupProcessor::runInMultipleThread(const cpp2::LookupIndexRequest& req) { resultDataSet_.append(std::move(partResults_[j])); } } + if (limit_ > 0) { + resultDataSet_.rows = TopKNode::topK(resultDataSet_.rows, orderBy_, limit_); + } // when run each part concurrently, we need to dedup again. if (!deDupColPos_.empty()) { DeDupNode::dedup(resultDataSet_.rows, deDupColPos_); diff --git a/src/storage/test/IndexScanLimitTest.cpp b/src/storage/test/IndexScanLimitTest.cpp index a73e0186a80..e87c83c8bcc 100644 --- a/src/storage/test/IndexScanLimitTest.cpp +++ b/src/storage/test/IndexScanLimitTest.cpp @@ -225,7 +225,7 @@ TEST_F(IndexScanLimitTest, LookupTagIndexLimit) { // limit == 0 { - req.set_limit(0); + req.indices_ref().value().set_limit(0); auto* processor = LookupProcessor::instance(storageEnv_.get(), nullptr, nullptr); auto fut = processor->getFuture(); processor->process(req); @@ -236,7 +236,7 @@ TEST_F(IndexScanLimitTest, LookupTagIndexLimit) { // limit == 1 { - req.set_limit(1); + req.indices_ref().value().set_limit(1); auto* processor = LookupProcessor::instance(storageEnv_.get(), nullptr, nullptr); auto fut = processor->getFuture(); processor->process(req); @@ -247,7 +247,7 @@ TEST_F(IndexScanLimitTest, LookupTagIndexLimit) { // limit 5 by each part { - req.set_limit(5); + req.indices_ref().value().set_limit(5); auto* processor = LookupProcessor::instance(storageEnv_.get(), nullptr, nullptr); auto fut = processor->getFuture(); processor->process(req); @@ -258,7 +258,7 @@ TEST_F(IndexScanLimitTest, LookupTagIndexLimit) { // limit 5 by each part through IndexScanNode->DataNode { - req.set_limit(5); + req.indices_ref().value().set_limit(5); cpp2::IndexColumnHint columnHint; columnHint.set_begin_value(Value(111)); columnHint.set_column_name("col1"); @@ -278,7 +278,7 @@ TEST_F(IndexScanLimitTest, LookupTagIndexLimit) { // limit 5 by each part through IndexScanNode->DataNode->FilterNode { - req.set_limit(5); + req.indices_ref().value().set_limit(5); cpp2::IndexColumnHint columnHint; columnHint.set_begin_value(Value(111)); columnHint.set_column_name("col1"); @@ -330,7 +330,7 @@ TEST_F(IndexScanLimitTest, LookupEdgeIndexLimit) { // limit == 0 { - req.set_limit(0); + req.indices_ref().value().set_limit(0); auto* processor = LookupProcessor::instance(storageEnv_.get(), nullptr, nullptr); auto fut = processor->getFuture(); processor->process(req); @@ -341,7 +341,7 @@ TEST_F(IndexScanLimitTest, LookupEdgeIndexLimit) { // limit == 1 { - req.set_limit(1); + req.indices_ref().value().set_limit(1); auto* processor = LookupProcessor::instance(storageEnv_.get(), nullptr, nullptr); auto fut = processor->getFuture(); processor->process(req); @@ -352,7 +352,7 @@ TEST_F(IndexScanLimitTest, LookupEdgeIndexLimit) { // limit 5 by each part { - req.set_limit(5); + req.indices_ref().value().set_limit(5); auto* processor = LookupProcessor::instance(storageEnv_.get(), nullptr, nullptr); auto fut = processor->getFuture(); processor->process(req); @@ -363,7 +363,7 @@ TEST_F(IndexScanLimitTest, LookupEdgeIndexLimit) { // limit 5 by each part through IndexScanNode->DataNode { - req.set_limit(5); + req.indices_ref().value().set_limit(5); cpp2::IndexColumnHint columnHint; columnHint.set_begin_value(Value(111)); columnHint.set_column_name("col1"); @@ -384,7 +384,7 @@ TEST_F(IndexScanLimitTest, LookupEdgeIndexLimit) { // limit 5 by each part through IndexScanNode->DataNode->FilterNode { - req.set_limit(5); + req.indices_ref().value().set_limit(5); cpp2::IndexColumnHint columnHint; columnHint.set_begin_value(Value(111)); columnHint.set_column_name("col1"); diff --git a/src/storage/test/StorageDAGTest.cpp b/src/storage/test/StorageDAGTest.cpp index e833f7a25fa..5d65d9cc3ec 100644 --- a/src/storage/test/StorageDAGTest.cpp +++ b/src/storage/test/StorageDAGTest.cpp @@ -28,7 +28,7 @@ TEST_F(StorageDAGTest, SimpleTest) { TEST_F(StorageDAGTest, ChainTest) { StoragePlan dag; - size_t lastIdx; + size_t lastIdx = 0; for (size_t i = 0; i < 10; i++) { auto node = std::make_unique>(folly::to(i)); if (i != 0) { diff --git a/tests/common/plan_differ.py b/tests/common/plan_differ.py index 1d722a40b4e..826b4712221 100644 --- a/tests/common/plan_differ.py +++ b/tests/common/plan_differ.py @@ -6,6 +6,7 @@ import re import json + class PlanDiffer: ID = "id" NAME = "name" @@ -125,12 +126,10 @@ def _is_subdict_nested(self, expect, resp): # The inner map cannot be empty if len(extracted_expected_dict) == 0: return None - # Unnested dict, push the first key into list - if extracted_expected_dict == expect: - key_list.append(list(expect.keys())[0]) extracted_resp_dict = {} - if len(key_list) == 1: + if extracted_expected_dict == expect: + # Unnested dict extracted_resp_dict = resp else: extracted_resp_dict = self._convert_jsonStr_to_dict(resp, key_list) @@ -139,7 +138,7 @@ def _is_subdict(small, big): return dict(big, **small) == big return _is_subdict(extracted_expected_dict, extracted_resp_dict) - + # resp: pair(key, jsonStr) def _convert_jsonStr_to_dict(self, resp, key_list): resp_json_str = '' @@ -194,7 +193,7 @@ def _validate_expect(self, rows, column_names): if self.OP_INFO not in column_names: self._err_msg = "Plan node operator info column is missing in expectde plan" return False - + id_idx_dict = {} # Check node id existence for i in range(len(rows)): diff --git a/tests/tck/features/lookup/LookUpTopN.feature b/tests/tck/features/lookup/LookUpTopN.feature new file mode 100644 index 00000000000..3938bde2f58 --- /dev/null +++ b/tests/tck/features/lookup/LookUpTopN.feature @@ -0,0 +1,222 @@ +# Copyright (c) 2021 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. +Feature: Push topN down IndexScan Rule + + Background: + Given a graph with space named "nba" + + Scenario: push topN down to IndexScan with order by + When profiling query: + """ + LOOKUP ON player | ORDER BY $-.VertexID | Limit 5 + """ + Then the result should be, in any order: + | VertexID | + | "Amar'e Stoudemire" | + | "Aron Baynes" | + | "Ben Simmons" | + | "Blake Griffin" | + | "Boris Diaw" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | TagIndexFullScan | 0 | {"orderBy": {"pos": "0"}} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON player | ORDER BY $-.VertexID | Limit 5 + """ + Then the result should be, in any order: + | VertexID | + | "Amar'e Stoudemire" | + | "Aron Baynes" | + | "Ben Simmons" | + | "Blake Griffin" | + | "Boris Diaw" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | TagIndexFullScan | 0 | {"limit": "5"} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON player WHERE player.age == 33 | ORDER BY $-.VertexID | Limit 3 + """ + Then the result should be, in any order: + | VertexID | + | "Chris Paul" | + | "Dwight Howard" | + | "LaMarcus Aldridge" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | TagIndexPrefixScan | 0 | {"orderBy": {"pos": "0"}} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON player WHERE player.age == 33 | ORDER BY $-.VertexID | Limit 3 + """ + Then the result should be, in any order: + | VertexID | + | "Chris Paul" | + | "Dwight Howard" | + | "LaMarcus Aldridge" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | TagIndexPrefixScan | 0 | {"limit": "3"} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON like | ORDER BY $-.SrcVID | Limit 5 + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | + | "Amar'e Stoudemire" | "Steve Nash" | 0 | + | "Aron Baynes" | "Tim Duncan" | 0 | + | "Ben Simmons" | "Joel Embiid" | 0 | + | "Blake Griffin" | "Chris Paul" | 0 | + | "Boris Diaw" | "Tony Parker" | 0 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | EdgeIndexFullScan | 0 | {"orderBy": {"pos": "0"}} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON like | ORDER BY $-.SrcVID | Limit 5 + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | + | "Amar'e Stoudemire" | "Steve Nash" | 0 | + | "Aron Baynes" | "Tim Duncan" | 0 | + | "Ben Simmons" | "Joel Embiid" | 0 | + | "Blake Griffin" | "Chris Paul" | 0 | + | "Boris Diaw" | "Tony Parker" | 0 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | EdgeIndexFullScan | 0 | {"limit": "5"} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON like WHERE like.likeness == 90 | ORDER BY $-.SrcVID | Limit 5 + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | + | "Amar'e Stoudemire" | "Steve Nash" | 0 | + | "Carmelo Anthony" | "LeBron James" | 0 | + | "Carmelo Anthony" | "Chris Paul" | 0 | + | "Carmelo Anthony" | "Dwyane Wade" | 0 | + | "Chris Paul" | "LeBron James" | 0 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | EdgeIndexPrefixScan | 0 | {"orderBy": {"pos": "0"}} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON like WHERE like.likeness == 90 | ORDER BY $-.SrcVID | Limit 5 + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | + | "Amar'e Stoudemire" | "Steve Nash" | 0 | + | "Carmelo Anthony" | "LeBron James" | 0 | + | "Carmelo Anthony" | "Chris Paul" | 0 | + | "Carmelo Anthony" | "Dwyane Wade" | 0 | + | "Chris Paul" | "LeBron James" | 0 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | TopN | 11 | | + | 11 | EdgeIndexPrefixScan | 0 | {"limit": "5"} | + | 0 | Start | | | + + Scenario: push topn down to IndexScan only limit + When profiling query: + """ + LOOKUP ON player | Limit 5 + """ + Then the result should be, in any order: + | VertexID | + | "Amar'e Stoudemire" | + | "Aron Baynes" | + | "Ben Simmons" | + | "Blake Griffin" | + | "Boris Diaw" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 4 | DataCollect | 7 | | + | 7 | Project | 8 | | + | 8 | Limit | 9 | | + | 9 | TagIndexFullScan | 0 | {"limit": "5"} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON player WHERE player.age == 33 | Limit 3 + """ + Then the result should be, in any order: + | VertexID | + | "Chris Paul" | + | "Dwight Howard" | + | "LaMarcus Aldridge" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 4 | DataCollect | 7 | | + | 7 | Project | 8 | | + | 8 | Limit | 9 | | + | 9 | TagIndexPrefixScan | 0 | {"limit": "3"} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON like | Limit 5 + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | + | "Aron Baynes" | "Tim Duncan" | 0 | + | "Ben Simmons" | "Joel Embiid" | 0 | + | "Blake Griffin" | "Chris Paul" | 0 | + | "Boris Diaw" | "Tim Duncan" | 0 | + | "Boris Diaw" | "Tony Parker" | 0 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | Limit | 11 | | + | 11 | EdgeIndexFullScan | 0 | {"limit": "5"} | + | 0 | Start | | | + When profiling query: + """ + LOOKUP ON like WHERE like.likeness == 90 | Limit 5 + """ + Then the result should be, in any order: + | SrcVID | DstVID | Ranking | + | "Amar'e Stoudemire" | "Steve Nash" | 0 | + | "Carmelo Anthony" | "LeBron James" | 0 | + | "Carmelo Anthony" | "Chris Paul" | 0 | + | "Carmelo Anthony" | "Dwyane Wade" | 0 | + | "Chris Paul" | "Carmelo Anthony" | 0 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 5 | DataCollect | 9 | | + | 9 | Project | 10 | | + | 10 | Limit | 11 | | + | 11 | EdgeIndexPrefixScan | 0 | {"limit": "5"} | + | 0 | Start | | |