From ca1098d91d614221d2ad4ab46f824c5065c7066e Mon Sep 17 00:00:00 2001 From: Pavel Velikhov Date: Sat, 3 Feb 2024 18:53:46 +0000 Subject: [PATCH 1/4] Optimized DPccp --- ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp | 14 +- ydb/core/kqp/opt/logical/kqp_opt_cbo.h | 1 + .../yql/core/cbo/cbo_optimizer_new.cpp | 86 +++++++- ydb/library/yql/core/cbo/cbo_optimizer_new.h | 45 ++++- ydb/library/yql/core/yql_cost_function.cpp | 13 +- ydb/library/yql/core/yql_statistics.cpp | 2 + ydb/library/yql/core/yql_statistics.h | 18 +- .../yql/dq/opt/dq_opt_join_cost_based.cpp | 183 ++++++++++++------ 8 files changed, 276 insertions(+), 86 deletions(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp b/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp index fed03ea383db..ef0604260eb0 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp @@ -90,9 +90,12 @@ bool IsLookupJoinApplicableDetailed(const std::shared_ptr left, std::shared_ptr right, const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, TKqpProviderContext& ctx) { Y_UNUSED(left); + Y_UNUSED(leftJoinKeys); auto rightStats = right->Stats; @@ -114,12 +117,7 @@ bool IsLookupJoinApplicable(std::shared_ptr left, } } - TVector joinKeys; - for( auto [leftJc, rightJc] : joinConditions ) { - joinKeys.emplace_back( rightJc.AttributeName); - } - - return IsLookupJoinApplicableDetailed(std::static_pointer_cast(right), joinKeys, ctx); + return IsLookupJoinApplicableDetailed(std::static_pointer_cast(right), rightJoinKeys, ctx); } } @@ -127,6 +125,8 @@ bool IsLookupJoinApplicable(std::shared_ptr left, bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptr& left, const std::shared_ptr& right, const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, EJoinAlgoType joinAlgo) { switch( joinAlgo ) { @@ -134,7 +134,7 @@ bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptrStats->Nrows > 10e3) { return false; } - return IsLookupJoinApplicable(left, right, joinConditions, *this); + return IsLookupJoinApplicable(left, right, joinConditions, leftJoinKeys, rightJoinKeys, *this); case EJoinAlgoType::DictJoin: return right->Stats->Nrows < 10e5; diff --git a/ydb/core/kqp/opt/logical/kqp_opt_cbo.h b/ydb/core/kqp/opt/logical/kqp_opt_cbo.h index 13b6b0200ec5..e43e0a33c19b 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_cbo.h +++ b/ydb/core/kqp/opt/logical/kqp_opt_cbo.h @@ -26,6 +26,7 @@ struct TKqpProviderContext : public NYql::IProviderContext { virtual bool IsJoinApplicable(const std::shared_ptr& left, const std::shared_ptr& right, const std::set>& joinConditions, + const TVector& leftJoinKeys, const TVector& rightJoinKeys, NYql::EJoinAlgoType joinAlgo) override; virtual double ComputeJoinCost(const NYql::TOptimizerStatistics& leftStats, const NYql::TOptimizerStatistics& rightStats, NYql::EJoinAlgoType joinAlgo) const override; diff --git a/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp b/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp index c7ce233d76f5..2adbc7e41bff 100644 --- a/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp +++ b/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp @@ -67,10 +67,14 @@ TJoinOptimizerNode::TJoinOptimizerNode(const std::shared_ptr IBaseOptimizerNode(JoinNodeType), LeftArg(left), RightArg(right), - JoinConditions(joinConditions), + JoinConditions(joinConditions), JoinType(joinType), JoinAlgo(joinAlgo) { IsReorderable = (JoinType==EJoinKind::InnerJoin) && (nonReorderable==false); + for (auto [l,r] : joinConditions ) { + LeftJoinKeys.push_back(l.AttributeName); + RightJoinKeys.push_back(r.AttributeName); + } } TVector TJoinOptimizerNode::Labels() { @@ -97,7 +101,85 @@ void TJoinOptimizerNode::Print(std::stringstream& stream, int ntabs) { stream << "\t"; } - stream << *Stats << "\n"; + if (Stats) { + stream << *Stats << "\n"; + } + + LeftArg->Print(stream, ntabs+1); + RightArg->Print(stream, ntabs+1); +} + +TJoinOptimizerNodeInternal::TJoinOptimizerNodeInternal(const std::shared_ptr& left, const std::shared_ptr& right, + const std::set>& joinConditions, const TVector& leftJoinKeys, + const TVector& rightJoinKeys, const EJoinKind joinType, const EJoinAlgoType joinAlgo, bool nonReorderable) : + IBaseOptimizerNode(JoinNodeType), + LeftArg(left), + RightArg(right), + JoinConditions(joinConditions), + LeftJoinKeys(leftJoinKeys), + RightJoinKeys(rightJoinKeys), + JoinType(joinType), + JoinAlgo(joinAlgo) { + IsReorderable = (JoinType==EJoinKind::InnerJoin) && (nonReorderable==false); + } + +TVector TJoinOptimizerNodeInternal::Labels() { + auto res = LeftArg->Labels(); + auto rightLabels = RightArg->Labels(); + res.insert(res.begin(),rightLabels.begin(),rightLabels.end()); + return res; +} + +std::shared_ptr ConvertFromInternal(const std::shared_ptr internal) { + auto left = internal->LeftArg; + auto right = internal->RightArg; + + if (left->Kind == EOptimizerNodeKind::JoinNodeType) { + left = ConvertFromInternal(std::static_pointer_cast(left)); + } + if (right->Kind == EOptimizerNodeKind::JoinNodeType) { + right = ConvertFromInternal(std::static_pointer_cast(right)); + } + + return std::make_shared(left, right, internal->JoinConditions, internal->JoinType, internal->JoinAlgo, !internal->IsReorderable); +} + +std::shared_ptr ConvertToInternal(const std::shared_ptr external) { + auto left = external->LeftArg; + auto right = external->RightArg; + + if (left->Kind == EOptimizerNodeKind::JoinNodeType) { + left = ConvertToInternal(std::static_pointer_cast(left)); + } + if (right->Kind == EOptimizerNodeKind::JoinNodeType) { + right = ConvertToInternal(std::static_pointer_cast(right)); + } + + return std::make_shared(left, right, external->JoinConditions, + external->LeftJoinKeys, external->RightJoinKeys, external->JoinType, external->JoinAlgo, !external->IsReorderable); +} + + +void TJoinOptimizerNodeInternal::Print(std::stringstream& stream, int ntabs) { + for (int i = 0; i < ntabs; i++){ + stream << "\t"; + } + + stream << "Join: (" << JoinType << ") "; + for (auto c : JoinConditions){ + stream << c.first.RelName << "." << c.first.AttributeName + << "=" << c.second.RelName << "." + << c.second.AttributeName << ", "; + } + stream << "\n"; + + for (int i = 0; i < ntabs; i++){ + stream << "\t"; + } + + if (Stats) { + stream << *Stats << "\n"; + } LeftArg->Print(stream, ntabs+1); RightArg->Print(stream, ntabs+1); diff --git a/ydb/library/yql/core/cbo/cbo_optimizer_new.h b/ydb/library/yql/core/cbo/cbo_optimizer_new.h index 256252241e76..172bb09a54d7 100644 --- a/ydb/library/yql/core/cbo/cbo_optimizer_new.h +++ b/ydb/library/yql/core/cbo/cbo_optimizer_new.h @@ -92,6 +92,8 @@ struct IProviderContext { virtual bool IsJoinApplicable(const std::shared_ptr& left, const std::shared_ptr& right, const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, EJoinAlgoType joinAlgo) = 0; }; @@ -111,11 +113,15 @@ struct TDummyProviderContext : public IProviderContext { bool IsJoinApplicable(const std::shared_ptr& left, const std::shared_ptr& right, const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, EJoinAlgoType joinAlgo) override { Y_UNUSED(left); Y_UNUSED(right); Y_UNUSED(joinConditions); + Y_UNUSED(leftJoinKeys); + Y_UNUSED(rightJoinKeys); Y_UNUSED(joinAlgo); return true; @@ -137,18 +143,51 @@ struct TDummyProviderContext : public IProviderContext { struct TJoinOptimizerNode : public IBaseOptimizerNode { std::shared_ptr LeftArg; std::shared_ptr RightArg; - std::set> JoinConditions; + const std::set> JoinConditions; + TVector LeftJoinKeys; + TVector RightJoinKeys; EJoinKind JoinType; EJoinAlgoType JoinAlgo; bool IsReorderable; - TJoinOptimizerNode(const std::shared_ptr& left, const std::shared_ptr& right, - const std::set>& joinConditions, const EJoinKind joinType, const EJoinAlgoType joinAlgo, bool nonReorderable=false); + TJoinOptimizerNode(const std::shared_ptr& left, + const std::shared_ptr& right, + const std::set>& joinConditions, + const EJoinKind joinType, + const EJoinAlgoType joinAlgo, + bool nonReorderable=false); virtual ~TJoinOptimizerNode() {} virtual TVector Labels(); virtual void Print(std::stringstream& stream, int ntabs=0); }; +struct TJoinOptimizerNodeInternal : public IBaseOptimizerNode { + std::shared_ptr LeftArg; + std::shared_ptr RightArg; + const std::set>& JoinConditions; + const TVector& LeftJoinKeys; + const TVector& RightJoinKeys; + EJoinKind JoinType; + EJoinAlgoType JoinAlgo; + bool IsReorderable; + + TJoinOptimizerNodeInternal(const std::shared_ptr& left, + const std::shared_ptr& right, + const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, + const EJoinKind joinType, + const EJoinAlgoType joinAlgo, + bool nonReorderable=false); + + virtual ~TJoinOptimizerNodeInternal() {} + virtual TVector Labels(); + virtual void Print(std::stringstream& stream, int ntabs=0); +}; + +std::shared_ptr ConvertFromInternal(const std::shared_ptr internal); +std::shared_ptr ConvertToInternal(const std::shared_ptr external); + struct IOptimizerNew { IProviderContext& Pctx; diff --git a/ydb/library/yql/core/yql_cost_function.cpp b/ydb/library/yql/core/yql_cost_function.cpp index dcf395ca408e..aae915e85d00 100644 --- a/ydb/library/yql/core/yql_cost_function.cpp +++ b/ydb/library/yql/core/yql_cost_function.cpp @@ -43,11 +43,13 @@ TOptimizerStatistics NYql::ComputeJoinStats(const TOptimizerStatistics& leftStat double newCard; EStatisticsType outputType; - TVector joinedTableKeys; + bool leftKeyColumns = false; + bool rightKeyColumns = false; + if (IsPKJoin(rightStats,rightJoinKeys)) { newCard = leftStats.Nrows; - joinedTableKeys = leftStats.KeyColumns; + leftKeyColumns = true; if (leftStats.Type == EStatisticsType::BaseTable){ outputType = EStatisticsType::FilteredFactTable; } else { @@ -56,7 +58,7 @@ TOptimizerStatistics NYql::ComputeJoinStats(const TOptimizerStatistics& leftStat } else if (IsPKJoin(leftStats,leftJoinKeys)) { newCard = rightStats.Nrows; - joinedTableKeys = rightStats.KeyColumns; + rightKeyColumns = true; if (rightStats.Type == EStatisticsType::BaseTable){ outputType = EStatisticsType::FilteredFactTable; } else { @@ -74,9 +76,11 @@ TOptimizerStatistics NYql::ComputeJoinStats(const TOptimizerStatistics& leftStat + newCard + leftStats.Cost + rightStats.Cost; - return TOptimizerStatistics(outputType, newCard, newNCols, cost, joinedTableKeys); + return TOptimizerStatistics(outputType, newCard, newNCols, cost, + leftKeyColumns ? leftStats.KeyColumns : ( rightKeyColumns ? rightStats.KeyColumns : TOptimizerStatistics::EmptyColumns)); } + TOptimizerStatistics NYql::ComputeJoinStats(const TOptimizerStatistics& leftStats, const TOptimizerStatistics& rightStats, const std::set>& joinConditions, EJoinAlgoType joinAlgo, const IProviderContext& ctx) { @@ -90,3 +94,4 @@ TOptimizerStatistics NYql::ComputeJoinStats(const TOptimizerStatistics& leftStat return ComputeJoinStats(leftStats, rightStats, leftJoinKeys, rightJoinKeys, joinAlgo, ctx); } + diff --git a/ydb/library/yql/core/yql_statistics.cpp b/ydb/library/yql/core/yql_statistics.cpp index 24033b50880b..18f4d463dcd4 100644 --- a/ydb/library/yql/core/yql_statistics.cpp +++ b/ydb/library/yql/core/yql_statistics.cpp @@ -17,3 +17,5 @@ TOptimizerStatistics& TOptimizerStatistics::operator+=(const TOptimizerStatistic Cost += other.Cost; return *this; } + +const TVector& TOptimizerStatistics::EmptyColumns = TVector(); diff --git a/ydb/library/yql/core/yql_statistics.h b/ydb/library/yql/core/yql_statistics.h index 77eff37837d4..a8969c182680 100644 --- a/ydb/library/yql/core/yql_statistics.h +++ b/ydb/library/yql/core/yql_statistics.h @@ -25,21 +25,19 @@ struct TOptimizerStatistics { double Nrows = 0; int Ncols = 0; double Cost; - TVector KeyColumns; - - TString Descr; - - TOptimizerStatistics() {} - TOptimizerStatistics(double nrows, int ncols): Nrows(nrows), Ncols(ncols) {} - TOptimizerStatistics(double nrows, int ncols, double cost): Nrows(nrows), Ncols(ncols), Cost(cost) {} - TOptimizerStatistics(EStatisticsType type, double nrows, int ncols, double cost): Type(type), Nrows(nrows), Ncols(ncols), Cost(cost) {} - TOptimizerStatistics(EStatisticsType type, double nrows, int ncols, double cost, TVector keyColumns): Type(type), Nrows(nrows), Ncols(ncols), Cost(cost), KeyColumns(keyColumns) {} - TOptimizerStatistics(double nrows,int ncols, double cost, TString descr): Nrows(nrows), Ncols(ncols), Cost(cost), Descr(descr) {} + const TVector& KeyColumns; + TOptimizerStatistics() : KeyColumns(EmptyColumns) {} + TOptimizerStatistics(double nrows, int ncols): Nrows(nrows), Ncols(ncols), KeyColumns(EmptyColumns) {} + TOptimizerStatistics(double nrows, int ncols, double cost): Nrows(nrows), Ncols(ncols), Cost(cost), KeyColumns(EmptyColumns) {} + TOptimizerStatistics(EStatisticsType type, double nrows, int ncols, double cost): Type(type), Nrows(nrows), Ncols(ncols), Cost(cost), KeyColumns(EmptyColumns) {} + TOptimizerStatistics(EStatisticsType type, double nrows, int ncols, double cost, const TVector& keyColumns): Type(type), Nrows(nrows), Ncols(ncols), Cost(cost), KeyColumns(keyColumns) {} TOptimizerStatistics& operator+=(const TOptimizerStatistics& other); bool Empty() const; friend std::ostream& operator<<(std::ostream& os, const TOptimizerStatistics& s); + + static const TVector& EmptyColumns; }; } diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 0ecc17f15323..b8ab359f98d4 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -40,16 +40,30 @@ struct TEdge { int From; int To; mutable std::set> JoinConditions; + mutable TVector LeftJoinKeys; + mutable TVector RightJoinKeys; + + TEdge(): From(-1), To(-1) {} TEdge(int f, int t): From(f), To(t) {} TEdge(int f, int t, std::pair cond): From(f), To(t) { JoinConditions.insert(cond); + BuildCondVectors(); } TEdge(int f, int t, std::set> conds): From(f), To(t), - JoinConditions(conds) {} + JoinConditions(conds) { + BuildCondVectors(); + } + void BuildCondVectors() { + for (auto [left, right] : JoinConditions) { + LeftJoinKeys.emplace_back(left.AttributeName); + RightJoinKeys.emplace_back(right.AttributeName); + } + } + bool operator==(const TEdge& other) const { return From==other.From && To==other.To; @@ -62,6 +76,8 @@ struct TEdge { return e.From + e.To; } }; + + static const struct TEdge ErrorEdge; }; /** @@ -94,77 +110,98 @@ void ComputeJoinConditions(const TCoEquiJoinTuple& joinTuple, /** * Create a new join and compute its statistics and cost */ -std::shared_ptr MakeJoin(std::shared_ptr left, +std::shared_ptr MakeJoin(std::shared_ptr left, std::shared_ptr right, const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, EJoinKind joinKind, EJoinAlgoType joinAlgo, IProviderContext& ctx) { - auto res = std::make_shared(left, right, joinConditions, joinKind, joinAlgo); - res->Stats = std::make_shared( ComputeJoinStats(*left->Stats, *right->Stats, joinConditions, joinAlgo, ctx)); + auto res = std::make_shared(left, right, joinConditions, leftJoinKeys, rightJoinKeys, joinKind, joinAlgo); + res->Stats = std::make_shared( ComputeJoinStats(*left->Stats, *right->Stats, leftJoinKeys, rightJoinKeys, joinAlgo, ctx)); return res; } +static const std::shared_ptr nullNode = std::shared_ptr(); /** * Iterate over all join algorithms and pick the best join that is applicable. * Also considers commuting joins */ -std::shared_ptr PickBestJoin(std::shared_ptr left, +std::shared_ptr PickBestJoin(std::shared_ptr left, std::shared_ptr right, const std::set>& leftJoinConditions, const std::set>& rightJoinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, IProviderContext& ctx) { - auto res = std::shared_ptr(); + double bestCost; + EJoinAlgoType bestAlgo; + bool bestJoinLeftRight = true; + bool bestJoinValid = false; for ( auto joinAlgo : AllJoinAlgos ) { - auto p1 = ctx.IsJoinApplicable(left, right, leftJoinConditions, joinAlgo) ? - MakeJoin(left, right, leftJoinConditions, EJoinKind::InnerJoin, joinAlgo, ctx) : - std::shared_ptr(); - auto p2 = ctx.IsJoinApplicable(right, left, rightJoinConditions, joinAlgo) ? - MakeJoin(right, left, rightJoinConditions, EJoinKind::InnerJoin, joinAlgo, ctx) : - std::shared_ptr(); - - if (p1) { - if (res) { - if (p1->Stats->Cost < res->Stats->Cost) { - res = p1; + if (ctx.IsJoinApplicable(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinAlgo)){ + auto cost = ComputeJoinStats(*left->Stats, *right->Stats, leftJoinKeys, rightJoinKeys, joinAlgo, ctx).Cost; + if (bestJoinValid){ + if (cost < bestCost) { + bestCost = cost; + bestAlgo = joinAlgo; + bestJoinLeftRight = true; } } else { - res = p1; + bestCost = cost; + bestAlgo = joinAlgo; + bestJoinLeftRight = true; + bestJoinValid = true; } } - if (p2) { - if (res) { - if (p2->Stats->Cost < res->Stats->Cost) { - res = p2; + + if (ctx.IsJoinApplicable(right, left, rightJoinConditions, rightJoinKeys, leftJoinKeys, joinAlgo)){ + auto cost = ComputeJoinStats(*right->Stats, *left->Stats, rightJoinKeys, leftJoinKeys, joinAlgo, ctx).Cost; + if (bestJoinValid){ + if (cost < bestCost) { + bestCost = cost; + bestAlgo = joinAlgo; + bestJoinLeftRight = false; } } else { - res = p2; + bestCost = cost; + bestAlgo = joinAlgo; + bestJoinLeftRight = false; + bestJoinValid = true; } } } - Y_ENSURE(res,"No join was chosen!"); - return res; + Y_ENSURE(bestJoinValid,"No join was chosen!"); + + if (bestJoinLeftRight) { + return MakeJoin(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, EJoinKind::InnerJoin, bestAlgo, ctx); + } else { + return MakeJoin(right, left, rightJoinConditions, rightJoinKeys, leftJoinKeys, EJoinKind::InnerJoin, bestAlgo, ctx); + } } /** * Iterate over all join algorithms and pick the best join that is applicable */ -std::shared_ptr PickBestNonReorderabeJoin(std::shared_ptr left, +std::shared_ptr PickBestNonReorderabeJoin(std::shared_ptr left, std::shared_ptr right, const std::set>& leftJoinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, EJoinKind joinKind, IProviderContext& ctx) { - auto res = std::shared_ptr(); + auto res = std::shared_ptr(); for ( auto joinAlgo : AllJoinAlgos ) { - auto p = ctx.IsJoinApplicable(left, right, leftJoinConditions, joinAlgo) ? - MakeJoin(left, right, leftJoinConditions, joinKind, joinAlgo, ctx) : - std::shared_ptr(); + auto p = ctx.IsJoinApplicable(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinAlgo) ? + MakeJoin(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinKind, joinAlgo, ctx) : + std::shared_ptr(); if (p) { if (res) { @@ -273,7 +310,7 @@ struct TGraph { // Find an edge that connects two subsets of graph's nodes // We are guaranteed to find a match - TEdge FindCrossingEdge(const std::bitset& S1, const std::bitset& S2) { + const TEdge& FindCrossingEdge(const std::bitset& S1, const std::bitset& S2) { for(int i = 0; i < NNodes; i++){ if (!S1[i]) { continue; @@ -290,7 +327,7 @@ struct TGraph { } } Y_ENSURE(false,"Connecting edge not found!"); - return TEdge(-1,-1); + return TEdge::ErrorEdge; } /** @@ -341,6 +378,10 @@ struct TGraph { Y_ABORT_UNLESS(maybeEdge1 != Edges.end() && maybeEdge2 != Edges.end()); maybeEdge1->JoinConditions.emplace(left, right); maybeEdge2->JoinConditions.emplace(right, left); + maybeEdge1->LeftJoinKeys.emplace_back(left.AttributeName); + maybeEdge1->RightJoinKeys.emplace_back(right.AttributeName); + maybeEdge2->LeftJoinKeys.emplace_back(right.AttributeName); + maybeEdge2->RightJoinKeys.emplace_back(left.AttributeName); } } } @@ -392,7 +433,7 @@ class TDPccpSolver { } // Run DPccp algorithm and produce the join tree in CBO's internal representation - std::shared_ptr Solve(); + std::shared_ptr Solve(); // Calculate the size of a dynamic programming table with a budget ui32 CountCC(ui32 budget); @@ -490,7 +531,7 @@ template std::bitset TDPccpSolver::Neighbors(const std::bitset& } // Run the entire DPccp algorithm and compute the optimal join tree -template std::shared_ptr TDPccpSolver::Solve() +template std::shared_ptr TDPccpSolver::Solve() { // Process singleton sets for (int i = NNodes-1; i >= 0; i--) { @@ -515,7 +556,7 @@ template std::shared_ptr TDPccpSolver::Solve() } Y_ENSURE(DpTable.contains(V), "Final relset not in dptable"); - return std::static_pointer_cast(DpTable[V]); + return std::static_pointer_cast(DpTable[V]); } /** @@ -629,9 +670,9 @@ template void TDPccpSolver::EmitCsgCmp(const std::bitset& S1, cons std::bitset joined = S1 | S2; - TEdge e1 = Graph.FindCrossingEdge(S1, S2); - TEdge e2 = Graph.FindCrossingEdge(S2, S1); - auto bestJoin = PickBestJoin(DpTable[S1], DpTable[S2], e1.JoinConditions, e2.JoinConditions, Pctx); + const TEdge& e1 = Graph.FindCrossingEdge(S1, S2); + const TEdge& e2 = Graph.FindCrossingEdge(S2, S1); + auto bestJoin = PickBestJoin(DpTable[S1], DpTable[S2], e1.JoinConditions, e2.JoinConditions, e1.LeftJoinKeys, e1.RightJoinKeys, Pctx); if (! DpTable.contains(joined)) { DpTable[joined] = bestJoin; @@ -923,6 +964,7 @@ std::shared_ptr ConvertToJoinTree(const TCoEquiJoinTuple& jo } std::set> joinConds; + size_t joinKeysCount = joinTuple.LeftKeys().Size() / 2; for (size_t i = 0; i < joinKeysCount; ++i) { size_t keyIndex = i * 2; @@ -942,15 +984,15 @@ std::shared_ptr ConvertToJoinTree(const TCoEquiJoinTuple& jo /** * Extract all non orderable joins from a plan is a post-order traversal order */ -void ExtractNonOrderables(std::shared_ptr joinTree, - TVector>& result) { +void ExtractNonOrderables(std::shared_ptr joinTree, + TVector>& result) { if (joinTree->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - auto left = static_pointer_cast(joinTree->LeftArg); + auto left = static_pointer_cast(joinTree->LeftArg); ExtractNonOrderables(left, result); } if (joinTree->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - auto right = static_pointer_cast(joinTree->RightArg); + auto right = static_pointer_cast(joinTree->RightArg); ExtractNonOrderables(right, result); } if (!joinTree->IsReorderable) @@ -964,7 +1006,7 @@ void ExtractNonOrderables(std::shared_ptr joinTree, * If we hit a non-orderable join type in recursion, we don't recurse into it and * add it as a relation */ -void ExtractRelsAndJoinConditions(const std::shared_ptr& joinTree, +void ExtractRelsAndJoinConditions(const std::shared_ptr& joinTree, TVector> & rels, std::set>& joinConditions) { if (!joinTree->IsReorderable){ @@ -977,14 +1019,14 @@ void ExtractRelsAndJoinConditions(const std::shared_ptr& joi } if (joinTree->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->LeftArg), rels, joinConditions); + ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->LeftArg), rels, joinConditions); } else { rels.emplace_back(joinTree->LeftArg); } if (joinTree->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->RightArg), rels, joinConditions); + ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->RightArg), rels, joinConditions); } else { rels.emplace_back(joinTree->RightArg); @@ -994,14 +1036,15 @@ void ExtractRelsAndJoinConditions(const std::shared_ptr& joi /** * Recursively computes statistics for a join tree */ -void ComputeStatistics(const std::shared_ptr& join, IProviderContext& ctx) { +void ComputeStatistics(const std::shared_ptr& join, IProviderContext& ctx) { if (join->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ComputeStatistics(static_pointer_cast(join->LeftArg), ctx); + ComputeStatistics(static_pointer_cast(join->LeftArg), ctx); } if (join->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ComputeStatistics(static_pointer_cast(join->RightArg), ctx); + ComputeStatistics(static_pointer_cast(join->RightArg), ctx); } - join->Stats = std::make_shared(ComputeJoinStats(*join->LeftArg->Stats, *join->RightArg->Stats, join->JoinConditions, EJoinAlgoType::DictJoin, ctx)); + join->Stats = std::make_shared(ComputeJoinStats(*join->LeftArg->Stats, *join->RightArg->Stats, + join->LeftJoinKeys, join->RightJoinKeys, EJoinAlgoType::DictJoin, ctx)); } /** @@ -1009,12 +1052,12 @@ void ComputeStatistics(const std::shared_ptr& join, IProvide * The root of the subtree that needs to be optimizer needs to be reorderable, otherwise we will * only update the statistics for it and return it unchanged */ -std::shared_ptr OptimizeSubtree(const std::shared_ptr& joinTree, ui32 maxDPccpDPTableSize, IProviderContext& ctx) { +std::shared_ptr OptimizeSubtree(const std::shared_ptr& joinTree, TGraph<128>& joinGraph, ui32 maxDPccpDPTableSize, IProviderContext& ctx) { if (!joinTree->IsReorderable) { - return PickBestNonReorderabeJoin(joinTree->LeftArg, joinTree->RightArg, joinTree->JoinConditions, joinTree->JoinType, ctx); + return PickBestNonReorderabeJoin(joinTree->LeftArg, joinTree->RightArg, joinTree->JoinConditions, + joinTree->LeftJoinKeys, joinTree->RightJoinKeys, joinTree->JoinType, ctx); } - TGraph<128> joinGraph; TVector> rels; std::set> joinConditions; @@ -1029,6 +1072,7 @@ std::shared_ptr OptimizeSubtree(const std::shared_ptr= 128) { ComputeStatistics(joinTree, ctx); + YQL_CLOG(TRACE, CoreDq) << "Too many rels"; return joinTree; } @@ -1065,7 +1109,7 @@ std::shared_ptr OptimizeSubtree(const std::shared_ptr result = solver.Solve(); + auto result = solver.Solve(); if (NYql::NLog::YqlLogger().NeedToLog(NYql::NLog::EComponent::ProviderKqp, NYql::NLog::ELevel::TRACE)) { std::stringstream str; @@ -1083,23 +1127,35 @@ class TOptimizerNativeNew: public IOptimizerNew { : IOptimizerNew(ctx), MaxDPccpDPTableSize(maxDPccpDPTableSize) { } std::shared_ptr JoinSearch(const std::shared_ptr& joinTree) override { + + auto _joinTree = ConvertToInternal(joinTree); + + TVector>> graphList; + // Traverse the join tree and generate a list of non-orderable joins in a post-order - TVector> nonOrderables; - ExtractNonOrderables(joinTree, nonOrderables); + TVector> nonOrderables; + ExtractNonOrderables(_joinTree, nonOrderables); // For all non-orderable joins, optimize the children for( auto join : nonOrderables ) { if (join->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - join->LeftArg = OptimizeSubtree(static_pointer_cast(join->LeftArg), MaxDPccpDPTableSize, Pctx); + auto g = std::make_shared>(); + join->LeftArg = OptimizeSubtree(static_pointer_cast(join->LeftArg), *g, MaxDPccpDPTableSize, Pctx); + graphList.emplace_back(g); } if (join->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - join->RightArg = OptimizeSubtree(static_pointer_cast(join->RightArg), MaxDPccpDPTableSize, Pctx); + auto g = std::make_shared>(); + join->RightArg = OptimizeSubtree(static_pointer_cast(join->RightArg), *g, MaxDPccpDPTableSize, Pctx); + graphList.emplace_back(g); } - join->Stats = std::make_shared(ComputeJoinStats(*join->LeftArg->Stats, *join->RightArg->Stats, join->JoinConditions, EJoinAlgoType::DictJoin, Pctx)); + join->Stats = std::make_shared(ComputeJoinStats(*join->LeftArg->Stats, *join->RightArg->Stats, + join->LeftJoinKeys, join->RightJoinKeys, EJoinAlgoType::DictJoin, Pctx)); } // Optimize the root - return OptimizeSubtree(joinTree, MaxDPccpDPTableSize, Pctx); + auto g = std::make_shared>(); + auto res = OptimizeSubtree(_joinTree, *g, MaxDPccpDPTableSize, Pctx); + return ConvertFromInternal(res); } const ui32 MaxDPccpDPTableSize; @@ -1152,6 +1208,13 @@ TExprBase DqOptimizeEquiJoinWithCosts(const TExprBase& node, TExprContext& ctx, // Generate an initial tree auto joinTree = ConvertToJoinTree(joinTuple,rels); + if (NYql::NLog::YqlLogger().NeedToLog(NYql::NLog::EComponent::ProviderKqp, NYql::NLog::ELevel::TRACE)) { + std::stringstream str; + str << "Converted join tree:\n"; + joinTree->Print(str); + YQL_CLOG(TRACE, CoreDq) << str.str(); + } + auto opt = TOptimizerNativeNew(providerCtx, maxDPccpDPTableSize); joinTree = opt.JoinSearch(joinTree); @@ -1173,7 +1236,7 @@ class TOptimizerNative: public IOptimizer { TOutput JoinSearch() override { auto dummyProviderCtx = TDummyProviderContext(); TDPccpSolver<128> solver(JoinGraph, Rels, dummyProviderCtx); - std::shared_ptr result = solver.Solve(); + std::shared_ptr result = ConvertFromInternal(solver.Solve()); if (Log) { std::stringstream str; str << "Join tree after cost based optimization:\n"; From 513a6f71d12c44b06e5a5858b403d4062385f19e Mon Sep 17 00:00:00 2001 From: Pavel Velikhov Date: Sat, 3 Feb 2024 19:10:24 +0000 Subject: [PATCH 2/4] Cleaned up new optimizer interface --- .../yql/core/cbo/cbo_optimizer_new.cpp | 76 -------------- ydb/library/yql/core/cbo/cbo_optimizer_new.h | 27 ----- .../yql/dq/opt/dq_opt_join_cost_based.cpp | 99 +++++++++++++++++++ 3 files changed, 99 insertions(+), 103 deletions(-) diff --git a/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp b/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp index 2adbc7e41bff..c125cb5bc009 100644 --- a/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp +++ b/ydb/library/yql/core/cbo/cbo_optimizer_new.cpp @@ -109,80 +109,4 @@ void TJoinOptimizerNode::Print(std::stringstream& stream, int ntabs) { RightArg->Print(stream, ntabs+1); } -TJoinOptimizerNodeInternal::TJoinOptimizerNodeInternal(const std::shared_ptr& left, const std::shared_ptr& right, - const std::set>& joinConditions, const TVector& leftJoinKeys, - const TVector& rightJoinKeys, const EJoinKind joinType, const EJoinAlgoType joinAlgo, bool nonReorderable) : - IBaseOptimizerNode(JoinNodeType), - LeftArg(left), - RightArg(right), - JoinConditions(joinConditions), - LeftJoinKeys(leftJoinKeys), - RightJoinKeys(rightJoinKeys), - JoinType(joinType), - JoinAlgo(joinAlgo) { - IsReorderable = (JoinType==EJoinKind::InnerJoin) && (nonReorderable==false); - } - -TVector TJoinOptimizerNodeInternal::Labels() { - auto res = LeftArg->Labels(); - auto rightLabels = RightArg->Labels(); - res.insert(res.begin(),rightLabels.begin(),rightLabels.end()); - return res; -} - -std::shared_ptr ConvertFromInternal(const std::shared_ptr internal) { - auto left = internal->LeftArg; - auto right = internal->RightArg; - - if (left->Kind == EOptimizerNodeKind::JoinNodeType) { - left = ConvertFromInternal(std::static_pointer_cast(left)); - } - if (right->Kind == EOptimizerNodeKind::JoinNodeType) { - right = ConvertFromInternal(std::static_pointer_cast(right)); - } - - return std::make_shared(left, right, internal->JoinConditions, internal->JoinType, internal->JoinAlgo, !internal->IsReorderable); -} - -std::shared_ptr ConvertToInternal(const std::shared_ptr external) { - auto left = external->LeftArg; - auto right = external->RightArg; - - if (left->Kind == EOptimizerNodeKind::JoinNodeType) { - left = ConvertToInternal(std::static_pointer_cast(left)); - } - if (right->Kind == EOptimizerNodeKind::JoinNodeType) { - right = ConvertToInternal(std::static_pointer_cast(right)); - } - - return std::make_shared(left, right, external->JoinConditions, - external->LeftJoinKeys, external->RightJoinKeys, external->JoinType, external->JoinAlgo, !external->IsReorderable); -} - - -void TJoinOptimizerNodeInternal::Print(std::stringstream& stream, int ntabs) { - for (int i = 0; i < ntabs; i++){ - stream << "\t"; - } - - stream << "Join: (" << JoinType << ") "; - for (auto c : JoinConditions){ - stream << c.first.RelName << "." << c.first.AttributeName - << "=" << c.second.RelName << "." - << c.second.AttributeName << ", "; - } - stream << "\n"; - - for (int i = 0; i < ntabs; i++){ - stream << "\t"; - } - - if (Stats) { - stream << *Stats << "\n"; - } - - LeftArg->Print(stream, ntabs+1); - RightArg->Print(stream, ntabs+1); -} - } // namespace NYql diff --git a/ydb/library/yql/core/cbo/cbo_optimizer_new.h b/ydb/library/yql/core/cbo/cbo_optimizer_new.h index 172bb09a54d7..e4c53da4fce6 100644 --- a/ydb/library/yql/core/cbo/cbo_optimizer_new.h +++ b/ydb/library/yql/core/cbo/cbo_optimizer_new.h @@ -161,33 +161,6 @@ struct TJoinOptimizerNode : public IBaseOptimizerNode { virtual void Print(std::stringstream& stream, int ntabs=0); }; -struct TJoinOptimizerNodeInternal : public IBaseOptimizerNode { - std::shared_ptr LeftArg; - std::shared_ptr RightArg; - const std::set>& JoinConditions; - const TVector& LeftJoinKeys; - const TVector& RightJoinKeys; - EJoinKind JoinType; - EJoinAlgoType JoinAlgo; - bool IsReorderable; - - TJoinOptimizerNodeInternal(const std::shared_ptr& left, - const std::shared_ptr& right, - const std::set>& joinConditions, - const TVector& leftJoinKeys, - const TVector& rightJoinKeys, - const EJoinKind joinType, - const EJoinAlgoType joinAlgo, - bool nonReorderable=false); - - virtual ~TJoinOptimizerNodeInternal() {} - virtual TVector Labels(); - virtual void Print(std::stringstream& stream, int ntabs=0); -}; - -std::shared_ptr ConvertFromInternal(const std::shared_ptr internal); -std::shared_ptr ConvertToInternal(const std::shared_ptr external); - struct IOptimizerNew { IProviderContext& Pctx; diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index b8ab359f98d4..c613cc114405 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -107,6 +107,105 @@ void ComputeJoinConditions(const TCoEquiJoinTuple& joinTuple, } } +struct TJoinOptimizerNodeInternal : public IBaseOptimizerNode { + std::shared_ptr LeftArg; + std::shared_ptr RightArg; + const std::set>& JoinConditions; + const TVector& LeftJoinKeys; + const TVector& RightJoinKeys; + EJoinKind JoinType; + EJoinAlgoType JoinAlgo; + bool IsReorderable; + + TJoinOptimizerNodeInternal(const std::shared_ptr& left, + const std::shared_ptr& right, + const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, + const EJoinKind joinType, + const EJoinAlgoType joinAlgo, + bool nonReorderable=false); + + virtual ~TJoinOptimizerNodeInternal() {} + virtual TVector Labels(); + virtual void Print(std::stringstream& stream, int ntabs=0); +}; + +TJoinOptimizerNodeInternal::TJoinOptimizerNodeInternal(const std::shared_ptr& left, const std::shared_ptr& right, + const std::set>& joinConditions, const TVector& leftJoinKeys, + const TVector& rightJoinKeys, const EJoinKind joinType, const EJoinAlgoType joinAlgo, bool nonReorderable) : + IBaseOptimizerNode(JoinNodeType), + LeftArg(left), + RightArg(right), + JoinConditions(joinConditions), + LeftJoinKeys(leftJoinKeys), + RightJoinKeys(rightJoinKeys), + JoinType(joinType), + JoinAlgo(joinAlgo) { + IsReorderable = (JoinType==EJoinKind::InnerJoin) && (nonReorderable==false); + } + +TVector TJoinOptimizerNodeInternal::Labels() { + auto res = LeftArg->Labels(); + auto rightLabels = RightArg->Labels(); + res.insert(res.begin(),rightLabels.begin(),rightLabels.end()); + return res; +} + +std::shared_ptr ConvertFromInternal(const std::shared_ptr internal) { + auto left = internal->LeftArg; + auto right = internal->RightArg; + + if (left->Kind == EOptimizerNodeKind::JoinNodeType) { + left = ConvertFromInternal(std::static_pointer_cast(left)); + } + if (right->Kind == EOptimizerNodeKind::JoinNodeType) { + right = ConvertFromInternal(std::static_pointer_cast(right)); + } + + return std::make_shared(left, right, internal->JoinConditions, internal->JoinType, internal->JoinAlgo, !internal->IsReorderable); +} + +std::shared_ptr ConvertToInternal(const std::shared_ptr external) { + auto left = external->LeftArg; + auto right = external->RightArg; + + if (left->Kind == EOptimizerNodeKind::JoinNodeType) { + left = ConvertToInternal(std::static_pointer_cast(left)); + } + if (right->Kind == EOptimizerNodeKind::JoinNodeType) { + right = ConvertToInternal(std::static_pointer_cast(right)); + } + + return std::make_shared(left, right, external->JoinConditions, + external->LeftJoinKeys, external->RightJoinKeys, external->JoinType, external->JoinAlgo, !external->IsReorderable); +} + + +void TJoinOptimizerNodeInternal::Print(std::stringstream& stream, int ntabs) { + for (int i = 0; i < ntabs; i++){ + stream << "\t"; + } + + stream << "Join: (" << JoinType << ") "; + for (auto c : JoinConditions){ + stream << c.first.RelName << "." << c.first.AttributeName + << "=" << c.second.RelName << "." + << c.second.AttributeName << ", "; + } + stream << "\n"; + + for (int i = 0; i < ntabs; i++){ + stream << "\t"; + } + + if (Stats) { + stream << *Stats << "\n"; + } + + LeftArg->Print(stream, ntabs+1); + RightArg->Print(stream, ntabs+1); +} /** * Create a new join and compute its statistics and cost */ From ae6e17a3ae6e1ce30fb619299113ff5af817589c Mon Sep 17 00:00:00 2001 From: Pavel Velikhov Date: Tue, 6 Feb 2024 20:24:50 +0000 Subject: [PATCH 3/4] Simplified the optimization a lot --- .../yql/dq/opt/dq_opt_join_cost_based.cpp | 136 +++++++++--------- 1 file changed, 70 insertions(+), 66 deletions(-) diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index c613cc114405..118070c3dcdd 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -152,36 +152,30 @@ TVector TJoinOptimizerNodeInternal::Labels() { return res; } -std::shared_ptr ConvertFromInternal(const std::shared_ptr internal) { - auto left = internal->LeftArg; - auto right = internal->RightArg; - - if (left->Kind == EOptimizerNodeKind::JoinNodeType) { - left = ConvertFromInternal(std::static_pointer_cast(left)); - } - if (right->Kind == EOptimizerNodeKind::JoinNodeType) { - right = ConvertFromInternal(std::static_pointer_cast(right)); +std::shared_ptr ConvertFromInternal(const std::shared_ptr internal) { + Y_ENSURE(internal->Kind == EOptimizerNodeKind::JoinNodeType); + + if (dynamic_cast(internal.get()) != nullptr) { + return std::static_pointer_cast(internal); } - return std::make_shared(left, right, internal->JoinConditions, internal->JoinType, internal->JoinAlgo, !internal->IsReorderable); -} - -std::shared_ptr ConvertToInternal(const std::shared_ptr external) { - auto left = external->LeftArg; - auto right = external->RightArg; + auto join = std::static_pointer_cast(internal); + + auto left = join->LeftArg; + auto right = join->RightArg; if (left->Kind == EOptimizerNodeKind::JoinNodeType) { - left = ConvertToInternal(std::static_pointer_cast(left)); + left = ConvertFromInternal(left); } if (right->Kind == EOptimizerNodeKind::JoinNodeType) { - right = ConvertToInternal(std::static_pointer_cast(right)); + right = ConvertFromInternal(right); } - return std::make_shared(left, right, external->JoinConditions, - external->LeftJoinKeys, external->RightJoinKeys, external->JoinType, external->JoinAlgo, !external->IsReorderable); + auto newJoin = std::make_shared(left, right, join->JoinConditions, join->JoinType, join->JoinAlgo, !join->IsReorderable); + newJoin->Stats = join->Stats; + return newJoin; } - void TJoinOptimizerNodeInternal::Print(std::stringstream& stream, int ntabs) { for (int i = 0; i < ntabs; i++){ stream << "\t"; @@ -206,10 +200,29 @@ void TJoinOptimizerNodeInternal::Print(std::stringstream& stream, int ntabs) { LeftArg->Print(stream, ntabs+1); RightArg->Print(stream, ntabs+1); } + /** * Create a new join and compute its statistics and cost */ -std::shared_ptr MakeJoin(std::shared_ptr left, +std::shared_ptr MakeJoin(std::shared_ptr left, + std::shared_ptr right, + const std::set>& joinConditions, + const TVector& leftJoinKeys, + const TVector& rightJoinKeys, + EJoinKind joinKind, + EJoinAlgoType joinAlgo, + bool nonReorderable, + IProviderContext& ctx) { + + auto res = std::make_shared(left, right, joinConditions, joinKind, joinAlgo, nonReorderable); + res->Stats = std::make_shared( ComputeJoinStats(*left->Stats, *right->Stats, leftJoinKeys, rightJoinKeys, joinAlgo, ctx)); + return res; +} + +/** + * Create a new join and compute its statistics and cost +*/ +std::shared_ptr MakeJoinInternal(std::shared_ptr left, std::shared_ptr right, const std::set>& joinConditions, const TVector& leftJoinKeys, @@ -223,7 +236,6 @@ std::shared_ptr MakeJoin(std::shared_ptr nullNode = std::shared_ptr(); /** * Iterate over all join algorithms and pick the best join that is applicable. * Also considers commuting joins @@ -278,16 +290,16 @@ std::shared_ptr PickBestJoin(std::shared_ptr PickBestNonReorderabeJoin(std::shared_ptr left, +std::shared_ptr PickBestNonReorderabeJoin(std::shared_ptr left, std::shared_ptr right, const std::set>& leftJoinConditions, const TVector& leftJoinKeys, @@ -295,27 +307,28 @@ std::shared_ptr PickBestNonReorderabeJoin(std::share EJoinKind joinKind, IProviderContext& ctx) { - auto res = std::shared_ptr(); + EJoinAlgoType bestJoinAlgo; + bool bestJoinValid = false; + double bestJoinCost; for ( auto joinAlgo : AllJoinAlgos ) { - auto p = ctx.IsJoinApplicable(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinAlgo) ? - MakeJoin(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinKind, joinAlgo, ctx) : - std::shared_ptr(); - - if (p) { - if (res) { - if (p->Stats->Cost < res->Stats->Cost) { - res = p; + if (ctx.IsJoinApplicable(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinAlgo)){ + auto cost = ComputeJoinStats(*right->Stats, *left->Stats, rightJoinKeys, leftJoinKeys, joinAlgo, ctx).Cost; + if (bestJoinValid) { + if (cost < bestJoinCost) { + bestJoinAlgo = joinAlgo; + bestJoinCost = cost; } } else { - res = p; + bestJoinAlgo = joinAlgo; + bestJoinCost = cost; + bestJoinValid = true; } } - } - Y_ENSURE(res,"No join was chosen!"); - return res; + Y_ENSURE(bestJoinValid,"No join was chosen!"); + return MakeJoin(left, right, leftJoinConditions, leftJoinKeys, rightJoinKeys, joinKind, bestJoinAlgo, true, ctx); } struct pair_hash { @@ -1083,15 +1096,15 @@ std::shared_ptr ConvertToJoinTree(const TCoEquiJoinTuple& jo /** * Extract all non orderable joins from a plan is a post-order traversal order */ -void ExtractNonOrderables(std::shared_ptr joinTree, - TVector>& result) { +void ExtractNonOrderables(std::shared_ptr joinTree, + TVector>& result) { if (joinTree->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - auto left = static_pointer_cast(joinTree->LeftArg); + auto left = static_pointer_cast(joinTree->LeftArg); ExtractNonOrderables(left, result); } if (joinTree->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - auto right = static_pointer_cast(joinTree->RightArg); + auto right = static_pointer_cast(joinTree->RightArg); ExtractNonOrderables(right, result); } if (!joinTree->IsReorderable) @@ -1105,7 +1118,7 @@ void ExtractNonOrderables(std::shared_ptr joinTree, * If we hit a non-orderable join type in recursion, we don't recurse into it and * add it as a relation */ -void ExtractRelsAndJoinConditions(const std::shared_ptr& joinTree, +void ExtractRelsAndJoinConditions(const std::shared_ptr& joinTree, TVector> & rels, std::set>& joinConditions) { if (!joinTree->IsReorderable){ @@ -1118,14 +1131,14 @@ void ExtractRelsAndJoinConditions(const std::shared_ptrLeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->LeftArg), rels, joinConditions); + ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->LeftArg), rels, joinConditions); } else { rels.emplace_back(joinTree->LeftArg); } if (joinTree->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->RightArg), rels, joinConditions); + ExtractRelsAndJoinConditions(static_pointer_cast(joinTree->RightArg), rels, joinConditions); } else { rels.emplace_back(joinTree->RightArg); @@ -1135,12 +1148,12 @@ void ExtractRelsAndJoinConditions(const std::shared_ptr& join, IProviderContext& ctx) { +void ComputeStatistics(const std::shared_ptr& join, IProviderContext& ctx) { if (join->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ComputeStatistics(static_pointer_cast(join->LeftArg), ctx); + ComputeStatistics(static_pointer_cast(join->LeftArg), ctx); } if (join->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - ComputeStatistics(static_pointer_cast(join->RightArg), ctx); + ComputeStatistics(static_pointer_cast(join->RightArg), ctx); } join->Stats = std::make_shared(ComputeJoinStats(*join->LeftArg->Stats, *join->RightArg->Stats, join->LeftJoinKeys, join->RightJoinKeys, EJoinAlgoType::DictJoin, ctx)); @@ -1151,7 +1164,7 @@ void ComputeStatistics(const std::shared_ptr& join, * The root of the subtree that needs to be optimizer needs to be reorderable, otherwise we will * only update the statistics for it and return it unchanged */ -std::shared_ptr OptimizeSubtree(const std::shared_ptr& joinTree, TGraph<128>& joinGraph, ui32 maxDPccpDPTableSize, IProviderContext& ctx) { +std::shared_ptr OptimizeSubtree(const std::shared_ptr& joinTree, ui32 maxDPccpDPTableSize, IProviderContext& ctx) { if (!joinTree->IsReorderable) { return PickBestNonReorderabeJoin(joinTree->LeftArg, joinTree->RightArg, joinTree->JoinConditions, joinTree->LeftJoinKeys, joinTree->RightJoinKeys, joinTree->JoinType, ctx); @@ -1159,9 +1172,10 @@ std::shared_ptr OptimizeSubtree(const std::shared_pt TVector> rels; std::set> joinConditions; - ExtractRelsAndJoinConditions(joinTree, rels, joinConditions); + TGraph<128> joinGraph; + for (size_t i = 0; i < rels.size(); i++) { joinGraph.AddNode(i, rels[i]->Labels()); } @@ -1217,7 +1231,7 @@ std::shared_ptr OptimizeSubtree(const std::shared_pt YQL_CLOG(TRACE, CoreDq) << str.str(); } - return result; + return ConvertFromInternal(result); } class TOptimizerNativeNew: public IOptimizerNew { @@ -1227,34 +1241,24 @@ class TOptimizerNativeNew: public IOptimizerNew { std::shared_ptr JoinSearch(const std::shared_ptr& joinTree) override { - auto _joinTree = ConvertToInternal(joinTree); - - TVector>> graphList; - // Traverse the join tree and generate a list of non-orderable joins in a post-order - TVector> nonOrderables; - ExtractNonOrderables(_joinTree, nonOrderables); + TVector> nonOrderables; + ExtractNonOrderables(joinTree, nonOrderables); // For all non-orderable joins, optimize the children for( auto join : nonOrderables ) { if (join->LeftArg->Kind == EOptimizerNodeKind::JoinNodeType) { - auto g = std::make_shared>(); - join->LeftArg = OptimizeSubtree(static_pointer_cast(join->LeftArg), *g, MaxDPccpDPTableSize, Pctx); - graphList.emplace_back(g); + join->LeftArg = OptimizeSubtree(static_pointer_cast(join->LeftArg), MaxDPccpDPTableSize, Pctx); } if (join->RightArg->Kind == EOptimizerNodeKind::JoinNodeType) { - auto g = std::make_shared>(); - join->RightArg = OptimizeSubtree(static_pointer_cast(join->RightArg), *g, MaxDPccpDPTableSize, Pctx); - graphList.emplace_back(g); + join->RightArg = OptimizeSubtree(static_pointer_cast(join->RightArg), MaxDPccpDPTableSize, Pctx); } join->Stats = std::make_shared(ComputeJoinStats(*join->LeftArg->Stats, *join->RightArg->Stats, join->LeftJoinKeys, join->RightJoinKeys, EJoinAlgoType::DictJoin, Pctx)); } // Optimize the root - auto g = std::make_shared>(); - auto res = OptimizeSubtree(_joinTree, *g, MaxDPccpDPTableSize, Pctx); - return ConvertFromInternal(res); + return OptimizeSubtree(joinTree, MaxDPccpDPTableSize, Pctx); } const ui32 MaxDPccpDPTableSize; From b90cfd3d37a4fa6adf8cd2215ed558f927a666f3 Mon Sep 17 00:00:00 2001 From: Pavel Velikhov Date: Wed, 7 Feb 2024 08:17:54 +0000 Subject: [PATCH 4/4] Added some comments --- .../yql/dq/opt/dq_opt_join_cost_based.cpp | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 118070c3dcdd..48c778c76e4c 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -107,6 +107,16 @@ void ComputeJoinConditions(const TCoEquiJoinTuple& joinTuple, } } +/** + * Internal Join nodes are used inside the CBO. They don't own join condition data structures + * and therefore avoid copying them during generation of candidate plans. + * + * These datastructures are owned by the query graph, so it is important to keep the graph around + * while internal nodes are being used. + * + * After join enumeration, internal nodes need to be converted to regular nodes, that own the data + * structures +*/ struct TJoinOptimizerNodeInternal : public IBaseOptimizerNode { std::shared_ptr LeftArg; std::shared_ptr RightArg; @@ -152,6 +162,13 @@ TVector TJoinOptimizerNodeInternal::Labels() { return res; } +/** + * Convert a tree of internal optimizer nodes to external nodes that own the data structures. + * + * The internal node tree can have references to external nodes (since some subtrees are optimized + * separately if the plan contains non-orderable joins). So we check the instances and if we encounter + * an external node, we return the whole subtree unchanged. +*/ std::shared_ptr ConvertFromInternal(const std::shared_ptr internal) { Y_ENSURE(internal->Kind == EOptimizerNodeKind::JoinNodeType); @@ -202,7 +219,7 @@ void TJoinOptimizerNodeInternal::Print(std::stringstream& stream, int ntabs) { } /** - * Create a new join and compute its statistics and cost + * Create a new external join node and compute its statistics and cost */ std::shared_ptr MakeJoin(std::shared_ptr left, std::shared_ptr right, @@ -220,7 +237,7 @@ std::shared_ptr MakeJoin(std::shared_ptr } /** - * Create a new join and compute its statistics and cost + * Create a new internal join node and compute its statistics and cost */ std::shared_ptr MakeJoinInternal(std::shared_ptr left, std::shared_ptr right, @@ -1008,6 +1025,9 @@ TExprBase RearrangeEquiJoinTree(TExprContext& ctx, const TCoEquiJoin& equiJoin, .Done(); } +/** + * Collects EquiJoin inputs with statistics for cost based optimization +*/ bool DqCollectJoinRelationsWithStats( TVector>& rels, TTypeAnnotationContext& typesCtx,