Enable SELECT from views #795
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| #include "rewrite_io_utils.h" | ||
|
|
||
| #include <ydb/core/kqp/provider/yql_kikimr_expr_nodes.h> | ||
| #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> | ||
| #include <ydb/library/yql/core/yql_expr_optimize.h> | ||
| #include <ydb/library/yql/providers/common/provider/yql_provider.h> | ||
| #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> | ||
| #include <ydb/library/yql/sql/sql.h> | ||
| #include <ydb/library/yql/utils/log/log.h> | ||
|
|
||
| namespace NYql { | ||
| namespace { | ||
|
|
||
| using namespace NNodes; | ||
|
|
||
| constexpr const char* QueryGraphNodeSignature = "SavedQueryGraph"; | ||
|
|
||
| NSQLTranslation::TTranslationSettings CreateViewTranslationSettings(const TString& cluster) { | ||
| NSQLTranslation::TTranslationSettings settings; | ||
|
|
||
| settings.DefaultCluster = cluster; | ||
| settings.ClusterMapping[cluster] = TString(NYql::KikimrProviderName); | ||
| settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; | ||
|
|
||
| return settings; | ||
| } | ||
|
|
||
| TExprNode::TPtr CompileViewQuery( | ||
| const TString& query, | ||
| TExprContext& ctx, | ||
| const TString& cluster | ||
| ) { | ||
| TAstParseResult queryAst; | ||
| queryAst = NSQLTranslation::SqlToYql(query, CreateViewTranslationSettings(cluster)); | ||
|
|
||
| ctx.IssueManager.AddIssues(queryAst.Issues); | ||
| if (!queryAst.IsOk()) { | ||
| return nullptr; | ||
| } | ||
|
|
||
| TExprNode::TPtr queryGraph; | ||
| if (!CompileExpr(*queryAst.Root, queryGraph, ctx, nullptr, nullptr)) { | ||
| return nullptr; | ||
| } | ||
|
|
||
| return queryGraph; | ||
| } | ||
|
|
||
| void AddChild(const TExprNode::TPtr& parent, const TExprNode::TPtr& newChild) { | ||
| auto childrenToChange = parent->ChildrenList(); | ||
| childrenToChange.emplace_back(newChild); | ||
| parent->ChangeChildrenInplace(std::move(childrenToChange)); | ||
| } | ||
|
|
||
| TExprNode::TPtr FindSavedQueryGraph(const TExprNode::TPtr& carrier) { | ||
| if (carrier->ChildrenSize() == 0) { | ||
| return nullptr; | ||
| } | ||
| auto lastChild = carrier->Children().back(); | ||
| return lastChild->IsCallable(QueryGraphNodeSignature) ? lastChild->ChildPtr(0) : TExprNode::TPtr(); | ||
| } | ||
|
|
||
| void SaveQueryGraph(const TExprNode::TPtr& carrier, TExprContext& ctx, const TExprNode::TPtr& payload) { | ||
| AddChild(carrier, ctx.NewCallable(payload->Pos(), QueryGraphNodeSignature, {payload})); | ||
| } | ||
|
|
||
| void InsertExecutionOrderDependencies( | ||
| TExprNode::TPtr& queryGraph, | ||
| const TExprNode::TPtr& worldBefore, | ||
| TExprContext& ctx | ||
| ) { | ||
| const auto initialWorldOfTheQuery = FindNode(queryGraph, [](const TExprNode::TPtr& node) { | ||
| return node->IsWorld(); | ||
| }); | ||
| if (!initialWorldOfTheQuery) { | ||
| return; | ||
| } | ||
| queryGraph = ctx.ReplaceNode(std::move(queryGraph), *initialWorldOfTheQuery, worldBefore); | ||
| } | ||
|
|
||
| bool CheckTopLevelness(const TExprNode::TPtr& candidateRead, const TExprNode::TPtr& queryGraph) { | ||
| THashSet<TExprNode::TPtr> readsInCandidateSubgraph; | ||
| VisitExpr(candidateRead, [&readsInCandidateSubgraph](const TExprNode::TPtr& node) { | ||
| if (node->IsCallable(ReadName)) { | ||
| readsInCandidateSubgraph.emplace(node); | ||
| } | ||
| return true; | ||
| }); | ||
|
|
||
| return !FindNode(queryGraph, [&readsInCandidateSubgraph](const TExprNode::TPtr& node) { | ||
| return node->IsCallable(ReadName) && !readsInCandidateSubgraph.contains(node); | ||
| }); | ||
| } | ||
|
|
||
| TExprNode::TPtr FindTopLevelRead(const TExprNode::TPtr& queryGraph) { | ||
| const TExprNode::TPtr* lastReadInTopologicalOrder = nullptr; | ||
| VisitExpr( | ||
| queryGraph, | ||
| nullptr, | ||
| [&lastReadInTopologicalOrder](const TExprNode::TPtr& node) { | ||
| if (node->IsCallable(ReadName)) { | ||
| lastReadInTopologicalOrder = &node; | ||
| } | ||
| return true; | ||
| } | ||
| ); | ||
|
|
||
| if (!lastReadInTopologicalOrder) { | ||
| return nullptr; | ||
| } | ||
|
|
||
| YQL_ENSURE(CheckTopLevelness(*lastReadInTopologicalOrder, queryGraph), | ||
|
Contributor
Author
There was a problem hiding this comment. Not sure if this is a good idea. This message is for developers, not for the user. However, I would like to check this assumption in the code somehow.
Contributor
Author
There was a problem hiding this comment. On the other hand, if I messed up in choosing the correct Read! node in this function, then the user will get an even more cryptic error message: "Failed to execute callable with name: ResWrite!, you possibly used cross provider/cluster operations or pulled not materialized result in refselect mode" This message appears whenever KQP failed to substitute some ResWrite! node to a TKiExecDataQuery! node, which usually happens when there are nodes with side-effects (like Read!) not seen on the path by the first child from the root of the query graph (see CheckTx function). |
||
| "Info for developers: assumption that there is only one top level Read! is wrong " | ||
| "for the expression graph of the query stored in the view:\n" | ||
| << queryGraph->Dump()); | ||
|
|
||
| return *lastReadInTopologicalOrder; | ||
| } | ||
|
|
||
| } | ||
|
|
||
| TExprNode::TPtr RewriteReadFromView( | ||
| const TExprNode::TPtr& node, | ||
| TExprContext& ctx, | ||
| const TString& query, | ||
| const TString& cluster | ||
| ) { | ||
| const TCoRead readNode(node->ChildPtr(0)); | ||
| const auto worldBeforeThisRead = readNode.World().Ptr(); | ||
|
|
||
| TExprNode::TPtr queryGraph = FindSavedQueryGraph(readNode.Ptr()); | ||
| if (!queryGraph) { | ||
| queryGraph = CompileViewQuery(query, ctx, cluster); | ||
| if (!queryGraph) { | ||
| ctx.AddError(TIssue(ctx.GetPosition(readNode.Pos()), | ||
| "The query stored in the view cannot be compiled.")); | ||
| return nullptr; | ||
| } | ||
| YQL_CLOG(TRACE, ProviderKqp) << "Expression graph of the query stored in the view:\n" | ||
| << NCommon::ExprToPrettyString(ctx, *queryGraph); | ||
|
|
||
| InsertExecutionOrderDependencies(queryGraph, worldBeforeThisRead, ctx); | ||
| SaveQueryGraph(readNode.Ptr(), ctx, queryGraph); | ||
| } | ||
|
|
||
| if (node->IsCallable(RightName)) { | ||
| return queryGraph; | ||
| } | ||
|
|
||
| const auto topLevelRead = FindTopLevelRead(queryGraph); | ||
|
Collaborator
There was a problem hiding this comment. I think you should at least combine InsertExecutionOrderDependencies and FindTopLevelRead methods. They serve the same purpose: get world topology roots and leafs.
Contributor
Author
There was a problem hiding this comment. One of the them (InsertExecutionOrderDependencies) is needed for both rewrites (rewrite of the Left! call to the read node and rewrite of the Right! call to the read node), while the other one (FindTopLevelRead) is needed only for the rewrite of the Left! call the read node. In addition, one of them does change the query graph, while the other is only searching for a node in it. I would like to keep them separated. |
||
| if (!topLevelRead) { | ||
|
Contributor
Author
There was a problem hiding this comment. It is actually pretty easy to have a query stored in a view that does not contain any reads. For example, SELECT 1 |
||
| return worldBeforeThisRead; | ||
| } | ||
| return Build<TCoLeft>(ctx, node->Pos()).Input(topLevelRead).Done().Ptr(); | ||
| } | ||
|
|
||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| #pragma once | ||
|
|
||
| #include <ydb/library/yql/ast/yql_expr.h> | ||
|
|
||
| namespace NYql { | ||
|
|
||
| TExprNode::TPtr RewriteReadFromView( | ||
| const TExprNode::TPtr& node, | ||
| TExprContext& ctx, | ||
| const TString& query, | ||
| const TString& cluster | ||
| ); | ||
|
|
||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.