Prune properties (#4523)

* delete useless code * support getProp only return vid & delete _src from edge * fix prune properties * enhance edge" * fix pathBuildExpr * fix union error * fix unwind * fix bicartesionproduct * add optimize appendvertice flag * fix test * fix format * add test * address comment * fix aggreate * fix conflict Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>
vesoft-inc · Aug 26, 2022 · decd853 · decd853
1 parent d575380
commit decd853
Show file tree

Hide file tree

Showing 18 changed files with 650 additions and 455 deletions.
diff --git a/src/graph/context/Iterator.cpp b/src/graph/context/Iterator.cpp
@@ -440,7 +440,7 @@ Value GetNeighborsIter::getVertex(const std::string& name) const {
     return Value::kNullValue;
   }
 
-  auto vidVal = getColumn(nebula::kVid);
+  auto vidVal = getColumn(0);
   if (UNLIKELY(!SchemaUtil::isValidVid(vidVal))) {
     return Value::kNullBadType;
   }
@@ -502,12 +502,13 @@ Value GetNeighborsIter::getEdge() const {
   edge.name = edgeName;
 
   auto type = getEdgeProp(edgeName, kType);
-  if (!type.isInt()) {
-    return Value::kNullBadType;
+  if (type.isInt()) {
+    edge.type = type.getInt();
+  } else {
+    edge.type = 0;
   }
-  edge.type = type.getInt();
 
-  auto& srcVal = getColumn(kVid);
+  auto& srcVal = getColumn(0);
   if (!SchemaUtil::isValidVid(srcVal)) {
     return Value::kNullBadType;
   }
@@ -520,10 +521,11 @@ Value GetNeighborsIter::getEdge() const {
   edge.dst = dstVal;
 
   auto& rank = getEdgeProp(edgeName, kRank);
-  if (!rank.isInt()) {
-    return Value::kNullBadType;
+  if (rank.isInt()) {
+    edge.ranking = rank.getInt();
+  } else {
+    edge.ranking = 0;
   }
-  edge.ranking = rank.getInt();
 
   auto& edgePropMap = currentDs_->edgePropsMap;
   auto edgeProp = edgePropMap.find(currentEdgeName());
@@ -535,7 +537,7 @@ Value GetNeighborsIter::getEdge() const {
   DCHECK_EQ(edgeNamePropList.size(), propList.size());
   for (size_t i = 0; i < propList.size(); ++i) {
     auto propName = edgeNamePropList[i];
-    if (propName == kSrc || propName == kDst || propName == kRank || propName == kType) {
+    if (propName == kDst || propName == kRank || propName == kType || propName == kSrc) {
       continue;
     }
     edge.props.emplace(edgeNamePropList[i], propList[i]);

diff --git a/src/graph/executor/query/AppendVerticesExecutor.cpp b/src/graph/executor/query/AppendVerticesExecutor.cpp
@@ -9,7 +9,7 @@
 using nebula::storage::StorageClient;
 using nebula::storage::StorageRpcResponse;
 using nebula::storage::cpp2::GetPropResponse;
-
+DECLARE_bool(optimize_appendvertices);
 namespace nebula {
 namespace graph {
 folly::Future<Status> AppendVerticesExecutor::execute() {
@@ -26,10 +26,12 @@ StatusOr<DataSet> AppendVerticesExecutor::buildRequestDataSet(const AppendVertic
 
 folly::Future<Status> AppendVerticesExecutor::appendVertices() {
   SCOPED_TIMER(&execTime_);
-
   auto *av = asNode<AppendVertices>(node());
-  StorageClient *storageClient = qctx()->getStorageClient();
+  if (FLAGS_optimize_appendvertices && av != nullptr && av->props() == nullptr) {
+    return handleNullProp(av);
+  }
 
+  StorageClient *storageClient = qctx()->getStorageClient();
   auto res = buildRequestDataSet(av);
   NG_RETURN_IF_ERROR(res);
   auto vertices = std::move(res).value();
@@ -69,6 +71,39 @@ folly::Future<Status> AppendVerticesExecutor::appendVertices() {
       });
 }
 
+Status AppendVerticesExecutor::handleNullProp(const AppendVertices *av) {
+  auto iter = ectx_->getResult(av->inputVar()).iter();
+  auto *src = av->src();
+
+  auto size = iter->size();
+  DataSet ds;
+  ds.colNames = av->colNames();
+  ds.rows.reserve(size);
+
+  QueryExpressionContext ctx(ectx_);
+  bool canBeMoved = movable(av->inputVars().front());
+
+  for (; iter->valid(); iter->next()) {
+    const auto &vid = src->eval(ctx(iter.get()));
+    if (vid.empty()) {
+      continue;
+    }
+    Vertex vertex;
+    vertex.vid = vid;
+    if (!av->trackPrevPath()) {
+      Row row;
+      row.values.emplace_back(std::move(vertex));
+      ds.rows.emplace_back(std::move(row));
+    } else {
+      Row row;
+      row = canBeMoved ? iter->moveRow() : *iter->row();
+      row.values.emplace_back(std::move(vertex));
+      ds.rows.emplace_back(std::move(row));
+    }
+  }
+  return finish(ResultBuilder().value(Value(std::move(ds))).build());
+}
+
 Status AppendVerticesExecutor::handleResp(
     storage::StorageRpcResponse<storage::cpp2::GetPropResponse> &&rpcResp) {
   auto result = handleCompleteness(rpcResp, FLAGS_accept_partial_success);

diff --git a/src/graph/executor/query/AppendVerticesExecutor.h b/src/graph/executor/query/AppendVerticesExecutor.h
@@ -28,6 +28,8 @@ class AppendVerticesExecutor final : public GetPropExecutor {
 
   Status handleResp(storage::StorageRpcResponse<storage::cpp2::GetPropResponse> &&rpcResp);
 
+  Status handleNullProp(const AppendVertices *av);
+
   folly::Future<Status> handleRespMultiJobs(
       storage::StorageRpcResponse<storage::cpp2::GetPropResponse> &&rpcResp);
 

diff --git a/src/graph/planner/match/MatchSolver.cpp b/src/graph/planner/match/MatchSolver.cpp
@@ -213,62 +213,6 @@ Expression* MatchSolver::makeIndexFilter(const std::string& label,
   return root;
 }
 
-void MatchSolver::extractAndDedupVidColumn(QueryContext* qctx,
-                                           Expression** initialExpr,
-                                           PlanNode* dep,
-                                           const std::string& inputVar,
-                                           SubPlan& plan) {
-  auto columns = qctx->objPool()->makeAndAdd<YieldColumns>();
-  auto* var = qctx->symTable()->getVar(inputVar);
-  Expression* vidExpr = initialExprOrEdgeDstExpr(qctx, initialExpr, var->colNames.back());
-  if (initialExpr) {
-    *initialExpr = nullptr;
-  }
-  columns->addColumn(new YieldColumn(vidExpr));
-  auto project = Project::make(qctx, dep, columns);
-  project->setInputVar(inputVar);
-  project->setColNames({kVid});
-  auto dedup = Dedup::make(qctx, project);
-  dedup->setColNames({kVid});
-
-  plan.root = dedup;
-}
-
-Expression* MatchSolver::initialExprOrEdgeDstExpr(QueryContext* qctx,
-                                                  Expression** initialExpr,
-                                                  const std::string& vidCol) {
-  if (initialExpr != nullptr && *initialExpr != nullptr) {
-    return *initialExpr;
-  } else {
-    return getEndVidInPath(qctx, vidCol);
-  }
-}
-
-Expression* MatchSolver::getEndVidInPath(QueryContext* qctx, const std::string& colName) {
-  auto* pool = qctx->objPool();
-  // expr: __Project_2[-1] => path
-  auto columnExpr = InputPropertyExpression::make(pool, colName);
-  // expr: endNode(path) => vn
-  auto args = ArgumentList::make(pool);
-  args->addArgument(columnExpr);
-  auto endNode = FunctionCallExpression::make(pool, "endNode", args);
-  // expr: en[_dst] => dst vid
-  auto vidExpr = ConstantExpression::make(pool, kVid);
-  return AttributeExpression::make(pool, endNode, vidExpr);
-}
-
-Expression* MatchSolver::getStartVidInPath(QueryContext* qctx, const std::string& colName) {
-  auto* pool = qctx->objPool();
-  // expr: __Project_2[0] => path
-  auto columnExpr = InputPropertyExpression::make(pool, colName);
-  // expr: startNode(path) => v1
-  auto args = ArgumentList::make(pool);
-  args->addArgument(columnExpr);
-  auto firstVertexExpr = FunctionCallExpression::make(pool, "startNode", args);
-  // expr: v1[_vid] => vid
-  return AttributeExpression::make(pool, firstVertexExpr, ConstantExpression::make(pool, kVid));
-}
-
 PlanNode* MatchSolver::filtPathHasSameEdge(PlanNode* input,
                                            const std::string& column,
                                            QueryContext* qctx) {
@@ -282,45 +226,6 @@ PlanNode* MatchSolver::filtPathHasSameEdge(PlanNode* input,
   return filter;
 }
 
-Status MatchSolver::appendFetchVertexPlan(const Expression* nodeFilter,
-                                          const SpaceInfo& space,
-                                          QueryContext* qctx,
-                                          Expression** initialExpr,
-                                          SubPlan& plan) {
-  return appendFetchVertexPlan(nodeFilter, space, qctx, initialExpr, plan.root->outputVar(), plan);
-}
-
-Status MatchSolver::appendFetchVertexPlan(const Expression* nodeFilter,
-                                          const SpaceInfo& space,
-                                          QueryContext* qctx,
-                                          Expression** initialExpr,
-                                          std::string inputVar,
-                                          SubPlan& plan) {
-  auto* pool = qctx->objPool();
-  // [Project && Dedup]
-  extractAndDedupVidColumn(qctx, initialExpr, plan.root, inputVar, plan);
-  auto srcExpr = InputPropertyExpression::make(pool, kVid);
-  // [Get vertices]
-  auto props = SchemaUtil::getAllVertexProp(qctx, space.id, true);
-  NG_RETURN_IF_ERROR(props);
-  auto gv = GetVertices::make(qctx, plan.root, space.id, srcExpr, std::move(props).value(), {});
-
-  PlanNode* root = gv;
-  if (nodeFilter != nullptr) {
-    auto* newFilter = MatchSolver::rewriteLabel2Vertex(qctx, nodeFilter);
-    root = Filter::make(qctx, root, newFilter);
-  }
-
-  // Normalize all columns to one
-  auto columns = pool->makeAndAdd<YieldColumns>();
-  auto pathExpr = PathBuildExpression::make(pool);
-  pathExpr->add(VertexExpression::make(pool));
-  columns->addColumn(new YieldColumn(pathExpr));
-  plan.root = Project::make(qctx, root, columns);
-  plan.root->setColNames({kPathStr});
-  return Status::OK();
-}
-
 static YieldColumn* buildVertexColumn(ObjectPool* pool, const std::string& alias) {
   return new YieldColumn(InputPropertyExpression::make(pool, alias), alias);
 }

diff --git a/src/graph/planner/match/MatchSolver.h b/src/graph/planner/match/MatchSolver.h
@@ -44,39 +44,10 @@ class MatchSolver final {
                                      QueryContext* qctx,
                                      bool isEdgeProperties = false);
 
-  static void extractAndDedupVidColumn(QueryContext* qctx,
-                                       Expression** initialExpr,
-                                       PlanNode* dep,
-                                       const std::string& inputVar,
-                                       SubPlan& plan);
-
-  static Expression* initialExprOrEdgeDstExpr(QueryContext* qctx,
-                                              Expression** initialExpr,
-                                              const std::string& vidCol);
-
-  static Expression* getEndVidInPath(QueryContext* qctx, const std::string& colName);
-
-  static Expression* getStartVidInPath(QueryContext* qctx, const std::string& colName);
-
   static PlanNode* filtPathHasSameEdge(PlanNode* input,
                                        const std::string& column,
                                        QueryContext* qctx);
 
-  static Status appendFetchVertexPlan(const Expression* nodeFilter,
-                                      const SpaceInfo& space,
-                                      QueryContext* qctx,
-                                      Expression** initialExpr,
-                                      SubPlan& plan);
-
-  // In 0 step left expansion case, the result of initial index scan
-  // will be passed as inputVar after right expansion is finished
-  static Status appendFetchVertexPlan(const Expression* nodeFilter,
-                                      const SpaceInfo& space,
-                                      QueryContext* qctx,
-                                      Expression** initialExpr,
-                                      std::string inputVar,
-                                      SubPlan& plan);
-
   // Build yield columns for match & shortestPath statement
   static void buildProjectColumns(QueryContext* qctx, Path& path, SubPlan& plan);
 };

diff --git a/src/graph/planner/ngql/PathPlanner.cpp b/src/graph/planner/ngql/PathPlanner.cpp
@@ -322,11 +322,12 @@ PlanNode* PathPlanner::buildVertexPlan(PlanNode* dep, const std::string& input)
   // col 0 of the project->output is [node...]
   auto* unwindExpr = ColumnExpression::make(pool, 0);
   auto* unwind = Unwind::make(qctx, project, unwindExpr);
+  unwind->setFromPipe(true);
   unwind->setColNames({"nodes"});
 
   // extract vid from vertex, col 0 is vertex
   auto idArgs = ArgumentList::make(pool);
-  idArgs->addArgument(ColumnExpression::make(pool, 1));
+  idArgs->addArgument(ColumnExpression::make(pool, 0));
   auto* src = FunctionCallExpression::make(pool, "id", idArgs);
   // get all vertexprop
   auto vertexProp = SchemaUtil::getAllVertexProp(qctx, pathCtx_->space.id, true);
@@ -355,23 +356,24 @@ PlanNode* PathPlanner::buildEdgePlan(PlanNode* dep, const std::string& input) {
   // col 0 of the project->output() is [edge...]
   auto* unwindExpr = ColumnExpression::make(pool, 0);
   auto* unwind = Unwind::make(qctx, project, unwindExpr);
+  unwind->setFromPipe(true);
   unwind->setColNames({"edges"});
 
   // extract src from edge
   auto srcArgs = ArgumentList::make(pool);
-  srcArgs->addArgument(ColumnExpression::make(pool, 1));
+  srcArgs->addArgument(ColumnExpression::make(pool, 0));
   auto* src = FunctionCallExpression::make(pool, "src", srcArgs);
   // extract dst from edge
   auto dstArgs = ArgumentList::make(pool);
-  dstArgs->addArgument(ColumnExpression::make(pool, 1));
+  dstArgs->addArgument(ColumnExpression::make(pool, 0));
   auto* dst = FunctionCallExpression::make(pool, "dst", dstArgs);
   // extract rank from edge
   auto rankArgs = ArgumentList::make(pool);
-  rankArgs->addArgument(ColumnExpression::make(pool, 1));
+  rankArgs->addArgument(ColumnExpression::make(pool, 0));
   auto* rank = FunctionCallExpression::make(pool, "rank", rankArgs);
   // type
   auto typeArgs = ArgumentList::make(pool);
-  typeArgs->addArgument(ColumnExpression::make(pool, 1));
+  typeArgs->addArgument(ColumnExpression::make(pool, 0));
   auto* type = FunctionCallExpression::make(pool, "typeid", typeArgs);
   // prepare edgetype
   auto edgeProp = SchemaUtil::getEdgeProps(qctx, pathCtx_->space, pathCtx_->over.edgeTypes, true);

diff --git a/src/graph/planner/plan/Algo.cpp b/src/graph/planner/plan/Algo.cpp
@@ -6,6 +6,7 @@
 #include "graph/planner/plan/Algo.h"
 
 #include "PlanNode.h"
+#include "graph/planner/plan/PlanNodeVisitor.h"
 #include "graph/util/ToJson.h"
 namespace nebula {
 namespace graph {
@@ -125,6 +126,10 @@ BiCartesianProduct::BiCartesianProduct(QueryContext* qctx, PlanNode* left, PlanN
   setColNames(lColNames);
 }
 
+void BiCartesianProduct::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 BiCartesianProduct::BiCartesianProduct(QueryContext* qctx)
     : BinaryInputNode(qctx, Kind::kBiCartesianProduct) {}
 

diff --git a/src/graph/planner/plan/Algo.h b/src/graph/planner/plan/Algo.h
@@ -313,6 +313,8 @@ class BiCartesianProduct final : public BinaryInputNode {
 
   PlanNode* clone() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
  private:
   friend ObjectPool;
 

diff --git a/src/graph/planner/plan/PlanNodeVisitor.h b/src/graph/planner/plan/PlanNodeVisitor.h
@@ -6,6 +6,7 @@
 #ifndef PLAN_PLANNODEVISITOR_H_
 #define PLAN_PLANNODEVISITOR_H_
 
+#include "graph/planner/plan/Algo.h"
 #include "graph/planner/plan/PlanNode.h"
 #include "graph/planner/plan/Query.h"
 
@@ -23,6 +24,9 @@ class PlanNodeVisitor {
   virtual void visit(Traverse *node) = 0;
   virtual void visit(AppendVertices *node) = 0;
   virtual void visit(BiJoin *node) = 0;
+  virtual void visit(Union *node) = 0;
+  virtual void visit(Unwind *node) = 0;
+  virtual void visit(BiCartesianProduct *node) = 0;
 };
 
 }  // namespace graph

diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp
@@ -321,6 +321,10 @@ void Union::cloneMembers(const Union& f) {
   SetOp::cloneMembers(f);
 }
 
+void Union::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 PlanNode* Intersect::clone() const {
   auto* newIntersect = Intersect::make(qctx_, nullptr, nullptr);
   newIntersect->cloneMembers(*this);
@@ -401,6 +405,10 @@ void Unwind::cloneMembers(const Unwind& p) {
   alias_ = p.alias();
 }
 
+void Unwind::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 std::unique_ptr<PlanNodeDescription> Sort::explain() const {
   auto desc = SingleInputNode::explain();
   addDescription("factors", folly::toJson(util::toJson(factorsString())), desc.get());