Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change limit to expression. #2878

Closed
wants to merge 8 commits into from
Closed

Change limit to expression. #2878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
85d625a
Change limit to expression.
Shylock-Hg Sep 16, 2021
1908e8a
Merge branch 'master' into feature/limit-expr
Shylock-Hg Sep 16, 2021
43a114f
Merge branch 'master' into feature/limit-expr
Shylock-Hg Sep 16, 2021
992f63b
Use method.
Shylock-Hg Sep 16, 2021
d939c46
Merge branch 'feature/limit-expr' of github.com:Shylock-Hg/nebula int…
Shylock-Hg Sep 16, 2021
0f74fe7
Eval with context in runtime.
Shylock-Hg Sep 17, 2021
9f895e8
Merge branch 'master' into feature/limit-expr
Shylock-Hg Sep 22, 2021
23fb973
Merge branch 'master' into feature/limit-expr
Shylock-Hg Sep 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/graph/executor/query/GetNeighborsExecutor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ folly::Future<Status> GetNeighborsExecutor::execute() {

time::Duration getNbrTime;
GraphStorageClient* storageClient = qctx_->getStorageClient();
QueryExpressionContext qec(qctx()->ectx());
return storageClient
->getNeighbors(gn_->space(),
qctx()->rctx()->session()->id(),
Expand All @@ -57,7 +58,7 @@ folly::Future<Status> GetNeighborsExecutor::execute() {
gn_->dedup(),
gn_->random(),
gn_->orderBy(),
gn_->limit(),
gn_->limit(qec),
gn_->filter())
.via(runner())
.ensure([this, getNbrTime]() {
Expand Down
3 changes: 2 additions & 1 deletion src/graph/executor/query/LimitExecutor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ folly::Future<Status> LimitExecutor::execute() {
ResultBuilder builder;
builder.value(result.valuePtr());
auto offset = limit->offset();
auto count = limit->count();
QueryExpressionContext qec(ectx_);
auto count = limit->count(qec);
auto size = iter->size();
if (size <= static_cast<size_t>(offset)) {
iter->clear();
Expand Down
4 changes: 4 additions & 0 deletions src/graph/optimizer/rule/LimitPushDownRule.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ StatusOr<OptRule::TransformResult> LimitPushDownRule::transform(
const auto proj = static_cast<const Project *>(projGroupNode->node());
const auto gn = static_cast<const GetNeighbors *>(gnGroupNode->node());

DCHECK(graph::ExpressionUtils::isEvaluableExpr(limit->countExpr()));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't use DCHECK to debug your code if it's the regular branch you need to handle.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This won't happen in correct state.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So why did you do the same check in line 54?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Handle exception scenes

if (!graph::ExpressionUtils::isEvaluableExpr(limit->countExpr())) {
return TransformResult::noTransform();
}
int64_t limitRows = limit->offset() + limit->count();
if (gn->limit() >= 0 && limitRows >= gn->limit()) {
return TransformResult::noTransform();
Expand Down
7 changes: 4 additions & 3 deletions src/graph/planner/plan/Query.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ std::unique_ptr<PlanNodeDescription> Explore::explain() const {
auto desc = SingleInputNode::explain();
addDescription("space", folly::to<std::string>(space_), desc.get());
addDescription("dedup", util::toJson(dedup_), desc.get());
addDescription("limit", folly::to<std::string>(limit_), desc.get());
addDescription(
"limit", folly::to<std::string>(limit_ == nullptr ? "" : limit_->toString()), desc.get());
std::string filter = filter_ == nullptr ? "" : filter_->toString();
addDescription("filter", filter, desc.get());
addDescription("orderBy", folly::toJson(util::toJson(orderBy_)), desc.get());
Expand Down Expand Up @@ -321,12 +322,12 @@ void Sort::cloneMembers(const Sort& p) {
std::unique_ptr<PlanNodeDescription> Limit::explain() const {
auto desc = SingleInputNode::explain();
addDescription("offset", folly::to<std::string>(offset_), desc.get());
addDescription("count", folly::to<std::string>(count_), desc.get());
addDescription("count", count_->toString(), desc.get());
return desc;
}

PlanNode* Limit::clone() const {
auto* newLimit = Limit::make(qctx_, nullptr);
auto* newLimit = Limit::make(qctx_, nullptr, -1, nullptr);
newLimit->cloneMembers(*this);
return newLimit;
}
Expand Down
72 changes: 67 additions & 5 deletions src/graph/planner/plan/Query.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,19 @@ class Explore : public SingleInputNode {

bool dedup() const { return dedup_; }

int64_t limit() const { return limit_; }
// Get the constant limit value
int64_t limit() const {
QueryExpressionContext ctx;
DCHECK(ExpressionUtils::isEvaluableExpr(limit_));
return DCHECK_NOTNULL(limit_)->eval(ctx).getInt();
}

// Get the limit value in runtime
int64_t limit(QueryExpressionContext& ctx) const {
return DCHECK_NOTNULL(limit_)->eval(ctx).getInt();
}

Expression* limitExpr() const { return limit_; }

const Expression* filter() const { return filter_; }

Expand All @@ -45,7 +57,9 @@ class Explore : public SingleInputNode {

void setDedup(bool dedup = true) { dedup_ = dedup; }

void setLimit(int64_t limit) { limit_ = limit; }
void setLimit(int64_t limit) { limit_ = ConstantExpression::make(qctx_->objPool(), limit); }

void setLimit(Expression* limit) { limit_ = limit; }

void setFilter(Expression* filter) { filter_ = filter; }

Expand All @@ -62,6 +76,21 @@ class Explore : public SingleInputNode {
int64_t limit,
Expression* filter,
std::vector<storage::cpp2::OrderBy> orderBy)
: SingleInputNode(qctx, kind, input),
space_(space),
dedup_(dedup),
limit_(ConstantExpression::make(qctx_->objPool(), limit)),
filter_(std::move(filter)),
orderBy_(std::move(orderBy)) {}

Explore(QueryContext* qctx,
Kind kind,
PlanNode* input,
GraphSpaceID space,
bool dedup,
Expression* limit,
Expression* filter,
std::vector<storage::cpp2::OrderBy> orderBy)
: SingleInputNode(qctx, kind, input),
space_(space),
dedup_(dedup),
Expand All @@ -77,7 +106,9 @@ class Explore : public SingleInputNode {
protected:
GraphSpaceID space_;
bool dedup_{false};
int64_t limit_{std::numeric_limits<int64_t>::max()};
// Use expression to get the limit value in runtime
// Now for the GetNeighbors/Limit in Loop
Expression* limit_{nullptr};
Expression* filter_{nullptr};
std::vector<storage::cpp2::OrderBy> orderBy_;
};
Expand Down Expand Up @@ -640,9 +671,34 @@ class Limit final : public SingleInputNode {
return qctx->objPool()->add(new Limit(qctx, input, offset, count));
}

static Limit* make(QueryContext* qctx,
PlanNode* input,
int64_t offset = -1,
Expression* count = nullptr) {
return qctx->objPool()->add(new Limit(qctx, input, offset, count));
}

int64_t offset() const { return offset_; }

int64_t count() const { return count_; }
// Get constant count value
int64_t count() const {
if (count_ == nullptr) {
return -1;
}
DCHECK(ExpressionUtils::isEvaluableExpr(count_));
QueryExpressionContext ctx;
return count_->eval(ctx).getInt();
}
Comment on lines +684 to +691
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this interface necessary?

Copy link
Contributor Author

@Shylock-Hg Shylock-Hg Sep 16, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Compatible.


// Get count in runtime
int64_t count(QueryExpressionContext& ctx) const {
if (count_ == nullptr) {
return -1;
}
return count_->eval(ctx).getInt();
}

const Expression* countExpr() const { return count_; }

PlanNode* clone() const override;
std::unique_ptr<PlanNodeDescription> explain() const override;
Expand All @@ -651,14 +707,20 @@ class Limit final : public SingleInputNode {
Limit(QueryContext* qctx, PlanNode* input, int64_t offset, int64_t count)
: SingleInputNode(qctx, Kind::kLimit, input) {
offset_ = offset;
count_ = ConstantExpression::make(qctx_->objPool(), count);
}

Limit(QueryContext* qctx, PlanNode* input, int64_t offset, Expression* count)
: SingleInputNode(qctx, Kind::kLimit, input) {
offset_ = offset;
count_ = count;
}

void cloneMembers(const Limit&);

private:
int64_t offset_{-1};
Copy link
Contributor

@czpmango czpmango Sep 16, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need change offset_ into an expression?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

offset won't push down

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is better to ensure consistency in use.

go from ....| offset abs(1)+3 limit abs(3)+1
go from ....| limit abs(3)+1,abs(3)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is used for new syntax for go ... limit [..]

int64_t count_{-1};
Expression* count_{nullptr};
};

/**
Expand Down