Add common planner

Signed-off-by: Xuanwo <github@xuanwo.io>

Author: Xuanwo

Cargo.lock

@@ -30,6 +30,7 @@ members = [
     "src/query/functions-v2",
     "src/query/legacy-parser",
     "src/query/management",
+    "src/query/planner",
     "src/query/pipeline/core",
     "src/query/pipeline/sinks",
     "src/query/pipeline/sources",

Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "common-planner"
+version = "0.1.0"
+authors = ["Databend Authors <opensource@datafuselabs.com>"]
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[dependencies]

src/query/planner/Cargo.toml

@@ -0,0 +1,25 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Databend Planner is the core part of Databend Query, it will:
+//!
+//! - Use `Parser` (provided by `common-ast`) to parse query into AST.
+//! - Use `Binder` to bind query into `LogicalPlan`
+//! - Use `Optimizer` to optimize `LogicalPlan` into `PhysicalPlan`
+//!
+//! After all the planners work, `Interpreter` will use `PhysicalPlan` to
+//! build pipelines, then our processes will produce result data blocks.
+
+mod metadata;
+pub use metadata::IndexType;

src/query/planner/src/lib.rs

@@ -0,0 +1,18 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Planner use [`usize`] as it's index type.
+///
+/// This type will be used across the whole planner.
+pub type IndexType = usize;

src/query/planner/src/metadata.rs

@@ -61,6 +61,7 @@ common-pipeline-core = { path = "../pipeline/core" }
 common-pipeline-sinks = { path = "../pipeline/sinks" }
 common-pipeline-sources = { path = "../pipeline/sources" }
 common-pipeline-transforms = { path = "../pipeline/transforms" }
+common-planner = { path = "../planner" }
 common-settings = { path = "../settings" }
 common-storage = { path = "../../common/storage" }
 common-storages-fuse = { path = "../storages/fuse" }

src/query/service/Cargo.toml

@@ -25,12 +25,12 @@ use common_datavalues::DataSchemaRefExt;
 use common_datavalues::DataTypeImpl;
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 use parking_lot::RwLock;
 
 use super::AggregateInfo;
 use crate::sql::normalize_identifier;
 use crate::sql::optimizer::SExpr;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Scalar;
 use crate::sql::NameResolutionContext;
 

src/query/service/src/sql/planner/binder/bind_context.rs

@@ -15,12 +15,12 @@
 use std::collections::HashMap;
 
 use common_exception::Result;
+use common_planner::IndexType;
 
 use crate::sql::binder::Binder;
 use crate::sql::binder::ColumnBinding;
 use crate::sql::optimizer::SExpr;
 use crate::sql::planner::semantic::GroupingChecker;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Aggregate;
 use crate::sql::plans::AggregateMode;
 use crate::sql::plans::BoundColumnRef;

src/query/service/src/sql/planner/binder/distinct.rs

@@ -18,6 +18,7 @@ use common_ast::ast::Indirection;
 use common_ast::ast::SelectTarget;
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::bind_context::NameResolutionResult;
 use crate::sql::binder::select::SelectItem;
@@ -31,7 +32,6 @@ use crate::sql::planner::binder::Binder;
 use crate::sql::planner::binder::ColumnBinding;
 use crate::sql::planner::semantic::normalize_identifier;
 use crate::sql::planner::semantic::GroupingChecker;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::BoundColumnRef;
 use crate::sql::plans::EvalScalar;
 use crate::sql::plans::Project;

src/query/service/src/sql/planner/binder/project.rs

@@ -21,6 +21,7 @@ use common_ast::ast::OrderByExpr;
 use common_ast::DisplayError;
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::bind_context::NameResolutionResult;
 use crate::sql::binder::scalar::ScalarBinder;
@@ -30,7 +31,6 @@ use crate::sql::binder::ColumnBinding;
 use crate::sql::normalize_identifier;
 use crate::sql::optimizer::SExpr;
 use crate::sql::planner::semantic::GroupingChecker;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::AggregateFunction;
 use crate::sql::plans::AndExpr;
 use crate::sql::plans::BoundColumnRef;

src/query/service/src/sql/planner/binder/sort.rs

@@ -31,6 +31,7 @@ use common_datavalues::prelude::*;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_legacy_planners::Expression;
+use common_planner::IndexType;
 
 use crate::sql::binder::scalar::ScalarBinder;
 use crate::sql::binder::Binder;
@@ -40,7 +41,6 @@ use crate::sql::binder::Visibility;
 use crate::sql::optimizer::SExpr;
 use crate::sql::planner::semantic::normalize_identifier;
 use crate::sql::planner::semantic::TypeChecker;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::ConstantExpr;
 use crate::sql::plans::LogicalGet;
 use crate::sql::plans::Scalar;

src/query/service/src/sql/planner/binder/table.rs

@@ -19,10 +19,10 @@ use std::sync::Arc;
 use common_ast::ast::Expr;
 use common_ast::ast::Literal;
 use common_datavalues::prelude::*;
+use common_planner::IndexType;
 use parking_lot::RwLock;
 
 use crate::sql::optimizer::ColumnSet;
-use crate::sql::planner::IndexType;
 use crate::storages::Table;
 
 pub static DUMMY_TABLE_INDEX: IndexType = IndexType::MAX;

src/query/service/src/sql/planner/metadata.rs

@@ -22,6 +22,7 @@ use std::sync::Arc;
 use common_catalog::table_context::TableContext;
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::cost::Cost;
 use super::cost::CostContext;
@@ -35,7 +36,6 @@ use crate::sql::optimizer::memo::Memo;
 use crate::sql::optimizer::rule::RuleSet;
 use crate::sql::optimizer::rule::TransformState;
 use crate::sql::optimizer::SExpr;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Operator;
 
 /// A cascades-style search engine to enumerate possible alternations of a relational expression and

src/query/service/src/sql/planner/optimizer/cascades/mod.rs

@@ -17,11 +17,11 @@ mod cost_model;
 use std::ops::Add;
 
 use common_exception::Result;
+use common_planner::IndexType;
 pub use cost_model::DefaultCostModel;
 
 use super::MExpr;
 use super::Memo;
-use crate::sql::planner::IndexType;
 
 #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)]
 pub struct Cost(pub f64);

src/query/service/src/sql/planner/optimizer/cost/mod.rs

@@ -13,10 +13,10 @@
 // limitations under the License.
 
 use common_exception::Result;
+use common_planner::IndexType;
 
 use crate::sql::optimizer::m_expr::MExpr;
 use crate::sql::optimizer::property::RelationalProperty;
-use crate::sql::planner::IndexType;
 
 /// `Group` is a set of logically equivalent relational expressions represented with `MExpr`.
 #[derive(Clone)]

src/query/service/src/sql/planner/optimizer/group.rs

@@ -20,6 +20,7 @@ use common_datavalues::DataTypeImpl;
 use common_datavalues::NullableType;
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use crate::sql::binder::wrap_cast;
 use crate::sql::binder::JoinPredicate;
@@ -30,7 +31,6 @@ use crate::sql::optimizer::heuristic::subquery_rewriter::UnnestResult;
 use crate::sql::optimizer::ColumnSet;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::SExpr;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Aggregate;
 use crate::sql::plans::AggregateFunction;
 use crate::sql::plans::AggregateMode;

src/query/service/src/sql/planner/optimizer/heuristic/decorrelate.rs

@@ -22,13 +22,13 @@ use common_datavalues::UInt64Type;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_functions::aggregates::AggregateFunctionFactory;
+use common_planner::IndexType;
 
 use crate::sql::binder::ColumnBinding;
 use crate::sql::binder::Visibility;
 use crate::sql::optimizer::ColumnSet;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::SExpr;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Aggregate;
 use crate::sql::plans::AggregateFunction;
 use crate::sql::plans::AggregateMode;

src/query/service/src/sql/planner/optimizer/heuristic/subquery_rewriter.rs

@@ -14,14 +14,14 @@
 
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::group::Group;
 use crate::sql::optimizer::memo::Memo;
 use crate::sql::optimizer::pattern_extractor::PatternExtractor;
 use crate::sql::optimizer::rule::RulePtr;
 use crate::sql::optimizer::rule::TransformState;
 use crate::sql::optimizer::SExpr;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Operator;
 use crate::sql::plans::RelOperator;
 

src/query/service/src/sql/planner/optimizer/m_expr.rs

@@ -16,13 +16,13 @@ use std::collections::HashSet;
 
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::RelExpr;
 use super::RelationalProperty;
 use crate::sql::optimizer::group::Group;
 use crate::sql::optimizer::m_expr::MExpr;
 use crate::sql::optimizer::s_expr::SExpr;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::RelOperator;
 
 /// `Memo` is a search space which memoize possible plans of a query.

src/query/service/src/sql/planner/optimizer/memo.rs

@@ -19,9 +19,9 @@ mod stat;
 use std::collections::HashSet;
 
 pub use builder::RelExpr;
+use common_planner::IndexType;
 pub use enforcer::require_property;
 
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Scalar;
 
 pub type ColumnSet = HashSet<IndexType>;

src/query/service/src/sql/planner/optimizer/property/mod.rs

@@ -14,11 +14,11 @@
 
 use common_exception::ErrorCode;
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::RelationalProperty;
 use crate::sql::optimizer::rule::AppliedRules;
 use crate::sql::optimizer::rule::RuleID;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::Operator;
 use crate::sql::plans::PatternPlan;
 use crate::sql::plans::RelOp;

src/query/service/src/sql/planner/optimizer/s_expr.rs

@@ -13,13 +13,13 @@
 // limitations under the License.#[derive(Clone, Debug)]
 
 use common_exception::Result;
+use common_planner::IndexType;
 
 use crate::sql::optimizer::ColumnSet;
 use crate::sql::optimizer::PhysicalProperty;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::RelationalProperty;
 use crate::sql::optimizer::RequiredProperty;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::LogicalOperator;
 use crate::sql::plans::Operator;
 use crate::sql::plans::PhysicalOperator;

src/query/service/src/sql/planner/plans/eval_scalar.rs

@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::JoinType;
 use crate::sql::optimizer::Distribution;
 use crate::sql::optimizer::PhysicalProperty;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::RequiredProperty;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::LogicalOperator;
 use crate::sql::plans::Operator;
 use crate::sql::plans::PhysicalOperator;

src/query/service/src/sql/planner/plans/hash_join.rs

@@ -14,12 +14,12 @@
 
 use common_catalog::table::TableStatistics;
 use common_exception::Result;
+use common_planner::IndexType;
 use itertools::Itertools;
 
 use crate::sql::optimizer::ColumnSet;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::RelationalProperty;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::LogicalOperator;
 use crate::sql::plans::Operator;
 use crate::sql::plans::PhysicalOperator;

src/query/service/src/sql/planner/plans/logical_get.rs

@@ -16,11 +16,11 @@ use std::fmt::Display;
 use std::fmt::Formatter;
 
 use common_exception::Result;
+use common_planner::IndexType;
 
 use super::ScalarExpr;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::RelationalProperty;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::LogicalOperator;
 use crate::sql::plans::Operator;
 use crate::sql::plans::PhysicalOperator;

src/query/service/src/sql/planner/plans/logical_join.rs

@@ -15,6 +15,7 @@
 use std::hash::Hash;
 
 use common_exception::Result;
+use common_planner::IndexType;
 use itertools::Itertools;
 
 use super::logical_get::Prewhere;
@@ -23,7 +24,6 @@ use crate::sql::optimizer::Distribution;
 use crate::sql::optimizer::PhysicalProperty;
 use crate::sql::optimizer::RelExpr;
 use crate::sql::optimizer::RequiredProperty;
-use crate::sql::planner::IndexType;
 use crate::sql::plans::LogicalOperator;
 use crate::sql::plans::Operator;
 use crate::sql::plans::PhysicalOperator;

src/query/service/src/sql/planner/plans/physical_scan.rs

@@ -21,11 +21,11 @@ use common_datavalues::DataValue;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_functions::scalars::FunctionFactory;
+use common_planner::IndexType;
 
 use crate::sql::binder::ColumnBinding;
 use crate::sql::optimizer::ColumnSet;
 use crate::sql::optimizer::SExpr;
-use crate::sql::planner::IndexType;
 
 pub trait ScalarExpr {
     /// Get return type and nullability