diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 5d6e3fa86cee5..68085a181252e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -605,7 +605,38 @@ export class BaseQuery { } } + buildSqlAndParamsTest(exportAnnotatedSql) { + if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { + if (this.externalPreAggregationQuery()) { // TODO performance + return this.externalQuery().buildSqlAndParams(exportAnnotatedSql); + } + } + const js_res = this.compilers.compiler.withQuery( + this, + () => this.cacheValue( + ['buildSqlAndParams', exportAnnotatedSql], + () => this.paramAllocator.buildSqlAndParams( + this.buildParamAnnotatedSql(), + exportAnnotatedSql, + this.shouldReuseParams + ), + { cache: this.queryCache } + ) + ); + console.log('js result: ', js_res[0]); + const rust = this.buildSqlAndParamsRust(exportAnnotatedSql); + console.log('rust result: ', rust[0]); + return js_res; + } + buildSqlAndParamsRust(exportAnnotatedSql) { + + + const order = this.options.order && R.pipe( + R.map((hash) => (!hash || !hash.id) ? null : hash), + R.reject(R.isNil), + )(this.options.order); + const queryParams = { measures: this.options.measures, dimensions: this.options.dimensions, @@ -614,12 +645,13 @@ export class BaseQuery { joinRoot: this.join.root, joinGraph: this.joinGraph, cubeEvaluator: this.cubeEvaluator, - order: this.options.order, + order: order, filters: this.options.filters, limit: this.options.limit ? this.options.limit.toString() : null, rowLimit: this.options.rowLimit ? this.options.rowLimit.toString() : null, offset: this.options.offset ? this.options.offset.toString() : null, baseTools: this, + ungrouped: this.options.ungrouped }; const res = nativeBuildSqlAndParams(queryParams); @@ -628,6 +660,10 @@ export class BaseQuery { return res; } + getAllocatedParams() { + return this.paramAllocator.getParams() + } + // FIXME helper for native generator, maybe should be moved entire to rust generateTimeSeries(granularity, dateRange) { return timeSeriesBase(granularity, dateRange); @@ -806,6 +842,7 @@ export class BaseQuery { } = this.fullKeyQueryAggregateMeasures(); if (!multipliedMeasures.length && !cumulativeMeasures.length && !multiStageMembers.length) { + console.log("!!!!! LLLOOOO!!!!"); return this.simpleQuery(); } @@ -1019,6 +1056,8 @@ export class BaseQuery { const allMemberChildren = this.collectAllMemberChildren(context); const memberToIsMultiStage = this.collectAllMultiStageMembers(allMemberChildren); + console.log("!!! measure to her ", measureToHierarchy); + const hasMultiStageMembers = (m) => { if (memberToIsMultiStage[m]) { return true; @@ -3288,7 +3327,8 @@ export class BaseQuery { gte: '{{ column }} >= {{ param }}', lt: '{{ column }} < {{ param }}', lte: '{{ column }} <= {{ param }}', - always_true: '1 == 1' + like_pattern: '{% if start_wild %}\'%\' || {% endif %}{{ value }}{% if end_wild %}|| \'%\'{% endif %}', + always_true: '1 = 1' }, quotes: { diff --git a/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts b/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts index 5dd8ae43d2c87..360dd90eecb35 100644 --- a/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts +++ b/packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts @@ -51,6 +51,10 @@ export class ParamAllocator { return `$${paramIndex}$`; } + public getParams() { + return this.params; + } + // eslint-disable-next-line no-unused-vars protected paramPlaceHolder(paramIndex) { return '?'; diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts index e8d6d6e4b2aac..342ed4ce26f68 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts @@ -453,7 +453,7 @@ describe('SQL Generation', () => { } }); - it('having filter with operator OR', async () => { + it('having filter with operator OR 1', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -486,7 +486,7 @@ describe('SQL Generation', () => { }] }); - console.log(query.buildSqlAndParams()); + console.log(query.buildSqlAndParamsTest()); return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { console.log(JSON.stringify(res)); @@ -648,7 +648,7 @@ describe('SQL Generation', () => { }); }); - it('where filter with operators OR & AND', async () => { + it('where filter with operators OR & AND 1', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index a595e0523dc17..4b21b0fde2dab 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -976,7 +976,7 @@ describe('SQL Generation', () => { console.log(query.buildSqlAndParams()); - return dbRunner.testQuery(query.buildSqlAndParams()).then(res => { + return dbRunner.testQuery(query.buildSqlAndParamsTest()).then(res => { console.log(JSON.stringify(res)); expect(res).toEqual( [{ visitor_checkins__revenue_per_checkin: '50' }] @@ -1721,7 +1721,7 @@ describe('SQL Generation', () => { ])); it( - 'contains filter', + 'contains filter 1', () => runQueryTest({ measures: [], dimensions: [ diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs index f08447d48c81f..2e0571cfa0fb5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs @@ -60,6 +60,7 @@ pub struct BaseQueryOptionsStatic { #[serde(rename = "rowLimit")] pub row_limit: Option, pub offset: Option, + pub ungrouped: Option, } #[nativebridge::native_bridge(BaseQueryOptionsStatic)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs index 5ade3438edc5c..a2c95592726aa 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs @@ -43,4 +43,5 @@ pub trait BaseTools { granularity: String, date_range: Vec, ) -> Result>, CubeError>; + fn get_allocated_params(&self) -> Result, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs index 9776ad592a37a..25f2c7c229191 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/filter.rs @@ -63,13 +63,16 @@ impl FilterItem { .items .iter() .map(|itm| itm.to_sql(templates, context.clone(), schema.clone())) - .collect::, _>>()?; - let result = if items_sql.is_empty() { - templates.always_true()? + .collect::, _>>()? + .into_iter() + .filter(|itm| !itm.is_empty()) + .collect::>(); + if items_sql.is_empty() { + "".to_string() } else { - items_sql.join(&operator) - }; - format!("({})", result) + let result = items_sql.join(&operator); + format!("({})", result) + } } FilterItem::Item(item) => { let sql = item.to_sql(context.clone(), schema)?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index fd1362ddad8ea..07e18b23eedac 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -59,23 +59,29 @@ impl BaseQuery { } fn build_sql_and_params_impl(&self) -> Result { + let nodes_factory = if self.request.ungrouped() { + SqlNodesFactory::new_ungroupped() + } else { + SqlNodesFactory::new() + }; + if self.request.is_simple_query()? { let planner = SimpleQueryPlanner::new( self.query_tools.clone(), self.request.clone(), - SqlNodesFactory::new(), + nodes_factory.clone(), ); planner.plan() } else { let multiplied_measures_query_planner = MultipliedMeasuresQueryPlanner::new( self.query_tools.clone(), self.request.clone(), - SqlNodesFactory::new(), + nodes_factory.clone(), ); let multi_stage_query_planner = MultiStageQueryPlanner::new(self.query_tools.clone(), self.request.clone()); let full_key_aggregate_planner = - FullKeyAggregateQueryPlanner::new(self.request.clone(), SqlNodesFactory::new()); + FullKeyAggregateQueryPlanner::new(self.request.clone(), nodes_factory.clone()); let mut subqueries = multiplied_measures_query_planner.plan_queries()?; let (multi_stage_ctes, multi_stage_subqueries) = multi_stage_query_planner.plan_queries()?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs index 249092a5774f2..d2751e8ce5572 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/base_filter.rs @@ -116,6 +116,12 @@ impl BaseFilter { FilterOperator::Gte => self.gte_where(&member_sql)?, FilterOperator::Lt => self.lt_where(&member_sql)?, FilterOperator::Lte => self.lte_where(&member_sql)?, + FilterOperator::Contains => self.contains_where(&member_sql)?, + FilterOperator::NotContains => self.not_contains_where(&member_sql)?, + FilterOperator::StartsWith => self.starts_with_where(&member_sql)?, + FilterOperator::NotStartsWith => self.not_starts_with_where(&member_sql)?, + FilterOperator::EndsWith => self.ends_with_where(&member_sql)?, + FilterOperator::NotEndsWith => self.not_ends_with_where(&member_sql)?, }; Ok(res) } @@ -243,6 +249,58 @@ impl BaseFilter { .lte(member_sql.to_string(), self.first_param()?) } + fn contains_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, false, true, true) + } + + fn not_contains_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, true, true, true) + } + + fn starts_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, false, false, true) + } + + fn not_starts_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, true, false, true) + } + + fn ends_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, false, true, false) + } + + fn not_ends_with_where(&self, member_sql: &str) -> Result { + self.like_or_where(member_sql, true, true, false) + } + + fn like_or_where( + &self, + member_sql: &str, + not: bool, + start_wild: bool, + end_wild: bool, + ) -> Result { + let values = self.filter_and_allocate_values(); + let like_parts = values + .into_iter() + .map(|v| { + self.templates + .ilike(member_sql, &v, start_wild, end_wild, not) + }) + .collect::, _>>()?; + let logical_symbol = if not { " AND " } else { " OR " }; + let null_check = if self.is_need_null_chek(not) { + self.templates.or_is_null_check(member_sql.to_string())? + } else { + "".to_string() + }; + Ok(format!( + "({}){}", + like_parts.join(logical_symbol), + null_check + )) + } + fn allocate_date_params(&self) -> Result<(String, String), CubeError> { if self.values.len() >= 2 { let from = if let Some(from_str) = &self.values[0] { @@ -345,13 +403,12 @@ impl BaseFilter { } fn allocate_param(&self, param: &str) -> String { - let index = self.query_tools.allocaate_param(param); - format!("${}$", index) + self.query_tools.allocate_param(param) } fn allocate_timestamp_param(&self, param: &str) -> String { - let index = self.query_tools.allocaate_param(param); - format!("${}$::timestamptz", index) + let placeholder = self.query_tools.allocate_param(param); + format!("{}::timestamptz", placeholder) } fn first_param(&self) -> Result { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs index d66caf4368928..30e0e39012813 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs @@ -15,6 +15,12 @@ pub enum FilterOperator { Gte, Lt, Lte, + Contains, + NotContains, + StartsWith, + NotStartsWith, + NotEndsWith, + EndsWith, } impl FromStr for FilterOperator { @@ -32,6 +38,12 @@ impl FromStr for FilterOperator { "gte" => Ok(Self::Gte), "lt" => Ok(Self::Lt), "lte" => Ok(Self::Lte), + "contains" => Ok(Self::Contains), + "notcontains" => Ok(Self::NotContains), + "startswith" => Ok(Self::StartsWith), + "notstartswith" => Ok(Self::NotStartsWith), + "endswith" => Ok(Self::EndsWith), + "notendswith" => Ok(Self::NotEndsWith), _ => Err(CubeError::user(format!("Unknown filter operator {}", s))), } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs index 6fe987015a4c5..47ffbd7268046 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/params_allocator.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; //const PARAMS_MATCH_REGEXP = /\$(\d+)\$/g; lazy_static! { - static ref PARAMS_MATCH_RE: Regex = Regex::new(r"\$(\d+)\$").unwrap(); + static ref PARAMS_MATCH_RE: Regex = Regex::new(r"\$_(\d+)_\$").unwrap(); } pub struct ParamsAllocator { params: Vec, @@ -16,9 +16,13 @@ impl ParamsAllocator { ParamsAllocator { params: Vec::new() } } - pub fn allocate_param(&mut self, name: &str) -> usize { + pub fn make_placeholder(&self, index: usize) -> String { + format!("$_{}_$", index) + } + + pub fn allocate_param(&mut self, name: &str) -> String { self.params.push(name.to_string()); - self.params.len() - 1 + self.make_placeholder(self.params.len() - 1) } pub fn get_params(&self) -> &Vec { @@ -28,18 +32,20 @@ impl ParamsAllocator { pub fn build_sql_and_params( &self, sql: &str, + native_allocated_params: Vec, should_reuse_params: bool, ) -> Result<(String, Vec), CubeError> { + let (sql, params) = self.add_native_allocated_params(sql, &native_allocated_params)?; let mut params_in_sql_order = Vec::new(); let mut param_index_map: HashMap = HashMap::new(); let result_sql = if should_reuse_params { PARAMS_MATCH_RE - .replace_all(sql, |caps: &Captures| { + .replace_all(&sql, |caps: &Captures| { let ind: usize = caps[1].to_string().parse().unwrap(); let new_index = if let Some(index) = param_index_map.get(&ind) { index.clone() } else { - params_in_sql_order.push(self.params[ind].clone()); + params_in_sql_order.push(params[ind].clone()); let index = params_in_sql_order.len(); param_index_map.insert(ind, index); index @@ -49,9 +55,9 @@ impl ParamsAllocator { .to_string() } else { PARAMS_MATCH_RE - .replace_all(sql, |caps: &Captures| { + .replace_all(&sql, |caps: &Captures| { let ind: usize = caps[1].to_string().parse().unwrap(); - params_in_sql_order.push(self.params[ind].clone()); + params_in_sql_order.push(params[ind].clone()); let index = params_in_sql_order.len(); format!("${}", index) //TODO get placeholder from js part }) @@ -59,4 +65,29 @@ impl ParamsAllocator { }; Ok((result_sql, params_in_sql_order)) } + + fn add_native_allocated_params( + &self, + sql: &str, + native_allocated_params: &Vec, + ) -> Result<(String, Vec), CubeError> { + lazy_static! { + static ref NATIVE_PARAMS_MATCH_RE: Regex = Regex::new(r"\$(\d+)\$").unwrap(); + } + + if native_allocated_params.is_empty() { + Ok((sql.to_string(), self.params.clone())) + } else { + let mut result_params = self.params.clone(); + let sql = NATIVE_PARAMS_MATCH_RE + .replace_all(sql, |caps: &Captures| { + let ind: usize = caps[1].to_string().parse().unwrap(); + let param = native_allocated_params[ind].clone(); + result_params.push(param); + self.make_placeholder(result_params.len() - 1) + }) + .to_string(); + Ok((sql, result_params)) + } + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs index d6bf872a608e0..7e22d32fa711f 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs @@ -33,21 +33,8 @@ impl FullKeyAggregateQueryPlanner { ))); } - let measures = self.query_properties.full_key_aggregate_measures()?; - - let inner_measures = measures - .multiplied_measures - .iter() - .chain(measures.multi_stage_measures.iter()) - .chain(measures.regular_measures.iter()) - .cloned() - .collect_vec(); - - let mut aggregate = self.outer_measures_join_full_key_aggregate( - &inner_measures, - &self.query_properties.measures(), - joins, - )?; + let mut aggregate = + self.outer_measures_join_full_key_aggregate(&self.query_properties.measures(), joins)?; if !ctes.is_empty() { aggregate.set_ctes(ctes.clone()); } @@ -57,7 +44,6 @@ impl FullKeyAggregateQueryPlanner { fn outer_measures_join_full_key_aggregate( &self, - _inner_measures: &Vec>, outer_measures: &Vec>, joins: Vec>, ) -> Result { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/join_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/join_planner.rs index 7d4aa92edf638..1c15442afcfa2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/join_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/join_planner.rs @@ -1,4 +1,5 @@ use super::CommonUtils; +use crate::cube_bridge::join_definition::JoinDefinition; use crate::cube_bridge::memeber_sql::MemberSql; use crate::plan::{From, JoinBuilder, JoinCondition}; use crate::planner::query_tools::QueryTools; @@ -25,6 +26,27 @@ impl JoinPlanner { alias_prefix: &Option, /*TODO dimensions for subqueries*/ ) -> Result { let join = self.query_tools.cached_data().join()?.clone(); + self.make_join_node_impl(alias_prefix, join) + } + + pub fn make_join_node_with_prefix_and_join_hints( + &self, + alias_prefix: &Option, /*TODO dimensions for subqueries*/ + join_hints: Vec, + ) -> Result { + let join = self.query_tools.join_graph().build_join(join_hints)?; + self.make_join_node_impl(alias_prefix, join) + } + + pub fn make_join_node(&self) -> Result { + self.make_join_node_with_prefix(&None) + } + + fn make_join_node_impl( + &self, + alias_prefix: &Option, + join: Rc, + ) -> Result { let root = self.utils.cube_from_path(join.static_data().root.clone())?; let joins = join.joins()?; if joins.items().is_empty() { @@ -56,10 +78,6 @@ impl JoinPlanner { } } - pub fn make_join_node(&self) -> Result { - self.make_join_node_with_prefix(&None) - } - fn compile_join_condition( &self, cube_name: &String, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 06533fcbed1f2..2ec6e71416834 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -295,6 +295,7 @@ impl MultiStageMemberQueryPlanner { None, None, true, + false, )?; let node_factory = if self.description.state().time_shifts().is_empty() { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs index 6ac49c628cea6..8ce29aa0ce1eb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multiplied_measures_query_planner.rs @@ -1,7 +1,10 @@ use super::{CommonUtils, JoinPlanner}; use crate::plan::{From, JoinBuilder, JoinCondition, Select, SelectBuilder}; use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::sql_nodes::SqlNodesFactory; +use crate::planner::sql_evaluator::collectors::{ + collect_cube_names, collect_join_hints, collect_join_hints_for_measures, +}; +use crate::planner::sql_evaluator::sql_nodes::{ungroupped_measure, SqlNodesFactory}; use crate::planner::BaseMember; use crate::planner::QueryProperties; use crate::planner::{BaseMeasure, VisitorContext}; @@ -10,6 +13,7 @@ use itertools::Itertools; use std::rc::Rc; pub struct MultipliedMeasuresQueryPlanner { + query_tools: Rc, query_properties: Rc, join_planner: JoinPlanner, common_utils: CommonUtils, @@ -23,6 +27,7 @@ impl MultipliedMeasuresQueryPlanner { context_factory: Rc, ) -> Self { Self { + query_tools: query_tools.clone(), join_planner: JoinPlanner::new(query_tools.clone()), common_utils: CommonUtils::new(query_tools.clone()), query_properties, @@ -66,6 +71,8 @@ impl MultipliedMeasuresQueryPlanner { let primary_keys_dimensions = self.common_utils.primary_keys_dimensions(key_cube_name)?; let keys_query = self.key_query(&primary_keys_dimensions, key_cube_name)?; let keys_query_alias = format!("keys"); + let should_build_join_for_measure_select = + self.check_should_build_join_for_measure_select(measures, key_cube_name)?; let mut join_builder = JoinBuilder::new_from_subselect(keys_query, keys_query_alias.clone()); @@ -73,16 +80,48 @@ impl MultipliedMeasuresQueryPlanner { let pk_cube = self.common_utils.cube_from_path(key_cube_name.clone())?; let pk_cube_alias = pk_cube.default_alias_with_prefix(&Some(format!("{key_cube_name}_key"))); - join_builder.left_join_cube( - pk_cube.clone(), - Some(pk_cube_alias.clone()), - JoinCondition::new_dimension_join( - keys_query_alias, - pk_cube_alias, - primary_keys_dimensions, - false, - ), - ); + let measures = if should_build_join_for_measure_select { + let mut top_measures = vec![]; + let mut ungroupped_measures = vec![]; + for meas in measures.iter() { + let ungropped_name = format!("{}_ungrouped", meas.name()); + let (top, ungrouped) = meas + .member_evaluator() + .clone() + .try_split_measure(ungropped_name) + .unwrap(); + top_measures.push(BaseMeasure::try_new(top, self.query_tools.clone())?.unwrap()); + ungroupped_measures + .push(BaseMeasure::try_new(ungrouped, self.query_tools.clone())?.unwrap()); + } + join_builder.left_join_subselect( + self.aggregate_subquery_measure_join( + key_cube_name, + &ungroupped_measures, + &primary_keys_dimensions, + )?, + pk_cube_alias.clone(), + JoinCondition::new_dimension_join( + keys_query_alias, + pk_cube_alias, + primary_keys_dimensions, + false, + ), + ); + top_measures + } else { + join_builder.left_join_cube( + pk_cube.clone(), + Some(pk_cube_alias.clone()), + JoinCondition::new_dimension_join( + keys_query_alias, + pk_cube_alias, + primary_keys_dimensions, + false, + ), + ); + measures.clone() + }; let mut select_builder = SelectBuilder::new( From::new_from_join(join_builder.build()), @@ -102,6 +141,55 @@ impl MultipliedMeasuresQueryPlanner { Ok(Rc::new(select_builder.build())) } + fn check_should_build_join_for_measure_select( + &self, + measures: &Vec>, + key_cube_name: &String, + ) -> Result { + for measure in measures.iter() { + let cubes = collect_cube_names(measure.member_evaluator())?; + let join_hints = collect_join_hints(measure.member_evaluator())?; + if cubes.iter().any(|cube| cube != key_cube_name) { + let measures_join = self.query_tools.join_graph().build_join(join_hints)?; + if *measures_join + .static_data() + .multiplication_factor + .get(key_cube_name) + .unwrap_or(&false) + { + return Err(CubeError::user(format!("{}' references cubes that lead to row multiplication. Please rewrite it using sub query.", measure.full_name()))); + } + return Ok(true); + } + } + Ok(false) + } + fn aggregate_subquery_measure_join( + &self, + key_cube_name: &String, + measures: &Vec>, + primary_keys_dimensions: &Vec>, + ) -> Result, CubeError> { + let join_hints = collect_join_hints_for_measures(measures)?; + let from = self + .join_planner + .make_join_node_with_prefix_and_join_hints(&None, join_hints)?; + let mut select_builder = SelectBuilder::new( + from, + VisitorContext::new( + None, + self.context_factory.ungroupped_measure_node_processor(), + ), + ); + for dim in primary_keys_dimensions.iter() { + select_builder.add_projection_member(dim, None, None); + } + for meas in measures.iter() { + select_builder.add_projection_member(&meas.clone().as_base_member(), None, None); + } + Ok(Rc::new(select_builder.build())) + } + fn regular_measures_subquery( &self, measures: &Vec>, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 63465f24c8ef7..ab29fc72db16d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -68,6 +68,7 @@ pub struct QueryProperties { offset: Option, query_tools: Rc, ignore_cumulative: bool, + ungrouped: bool, } impl QueryProperties { @@ -161,6 +162,7 @@ impl QueryProperties { } else { None }; + let ungrouped = options.static_data().ungrouped.unwrap_or(false); Ok(Rc::new(Self { measures, @@ -174,6 +176,7 @@ impl QueryProperties { offset, query_tools, ignore_cumulative: false, + ungrouped, })) } @@ -189,6 +192,7 @@ impl QueryProperties { row_limit: Option, offset: Option, ignore_cumulative: bool, + ungrouped: bool, ) -> Result, CubeError> { let order_by = if order_by.is_empty() { Self::default_order(&dimensions, &time_dimensions, &measures) @@ -208,6 +212,7 @@ impl QueryProperties { offset, query_tools, ignore_cumulative, + ungrouped, })) } @@ -252,6 +257,10 @@ impl QueryProperties { Self::default_order(&self.dimensions, &self.time_dimensions, &self.measures); } + pub fn ungrouped(&self) -> bool { + self.ungrouped + } + pub fn all_filters(&self) -> Option { let items = self .time_dimensions_filters @@ -326,15 +335,19 @@ impl QueryProperties { } pub fn group_by(&self) -> Vec { - self.dimensions - .iter() - .map(|f| Expr::Member(MemberExpression::new(f.clone(), None))) - .chain( - self.time_dimensions - .iter() - .map(|f| Expr::Member(MemberExpression::new(f.clone(), None))), - ) - .collect() + if self.ungrouped { + vec![] + } else { + self.dimensions + .iter() + .map(|f| Expr::Member(MemberExpression::new(f.clone(), None))) + .chain( + self.time_dimensions + .iter() + .map(|f| Expr::Member(MemberExpression::new(f.clone(), None))), + ) + .collect() + } } pub fn default_order( @@ -381,13 +394,14 @@ impl QueryProperties { } pub fn is_simple_query(&self) -> Result { - for member in self.all_members(false) { - match self.get_symbol_aggregate_type(&member.member_evaluator())? { - SymbolAggregateType::Regular => {} - _ => return Ok(false), - } + let full_aggregate_measure = self.full_key_aggregate_measures()?; + if full_aggregate_measure.multiplied_measures.is_empty() + && full_aggregate_measure.multi_stage_measures.is_empty() + { + Ok(true) + } else { + Ok(false) } - Ok(true) } pub fn should_use_time_series(&self) -> Result { @@ -403,33 +417,21 @@ impl QueryProperties { let mut result = FullKeyAggregateMeasures::default(); let measures = self.measures(); for m in measures.iter() { - match self.get_symbol_aggregate_type(m.member_evaluator())? { - SymbolAggregateType::Regular => result.regular_measures.push(m.clone()), - SymbolAggregateType::Multiplied => result.multiplied_measures.push(m.clone()), - SymbolAggregateType::MultiStage => result.multi_stage_measures.push(m.clone()), + if has_multi_stage_members(m.member_evaluator(), self.ignore_cumulative)? { + result.multi_stage_measures.push(m.clone()) + } else { + for item in + collect_multiplied_measures(self.query_tools.clone(), m.member_evaluator())? + { + if item.multiplied { + result.multiplied_measures.push(item.measure.clone()); + } else { + result.regular_measures.push(item.measure.clone()); + } + } } } Ok(result) } - - fn get_symbol_aggregate_type( - &self, - symbol: &Rc, - ) -> Result { - let symbol_type = if has_multi_stage_members(symbol, self.ignore_cumulative)? { - SymbolAggregateType::MultiStage - } else if let Some(multiple) = - collect_multiplied_measures(self.query_tools.clone(), symbol)? - { - if multiple.multiplied { - SymbolAggregateType::Multiplied - } else { - SymbolAggregateType::Regular - } - } else { - SymbolAggregateType::Regular - }; - Ok(symbol_type) - } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs index d20896c4bec23..632b457243556 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_tools.rs @@ -141,7 +141,7 @@ impl QueryTools { self.templates_render.clone() } - pub fn allocaate_param(&self, name: &str) -> usize { + pub fn allocate_param(&self, name: &str) -> String { self.params_allocator.borrow_mut().allocate_param(name) } pub fn get_allocated_params(&self) -> Vec { @@ -152,8 +152,11 @@ impl QueryTools { sql: &str, should_reuse_params: bool, ) -> Result<(String, Vec), CubeError> { - self.params_allocator - .borrow() - .build_sql_and_params(sql, should_reuse_params) + let native_allocated_params = self.base_tools.get_allocated_params()?; + self.params_allocator.borrow().build_sql_and_params( + sql, + native_allocated_params, + should_reuse_params, + ) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs new file mode 100644 index 0000000000000..5cfc6c2f81cef --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/cube_names_collector.rs @@ -0,0 +1,64 @@ +use crate::planner::sql_evaluator::{ + EvaluationNode, MemberSymbol, MemberSymbolType, TraversalVisitor, +}; +use cubenativeutils::CubeError; +use std::collections::HashSet; +use std::rc::Rc; + +pub struct CubeNamesCollector { + names: HashSet, +} + +impl CubeNamesCollector { + pub fn new() -> Self { + Self { + names: HashSet::new(), + } + } + + pub fn extract_result(self) -> Vec { + self.names.into_iter().collect() + } +} + +impl TraversalVisitor for CubeNamesCollector { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { + match node.symbol() { + MemberSymbolType::Dimension(e) => { + if e.owned_by_cube() { + self.names.insert(e.cube_name().clone()); + } + } + MemberSymbolType::Measure(e) => { + for filter_node in e.measure_filters() { + self.apply(filter_node, &())? + } + for order_by in e.measure_order_by() { + self.apply(order_by.evaluation_node(), &())? + } + if e.owned_by_cube() { + self.names.insert(e.cube_name().clone()); + } + } + MemberSymbolType::CubeName(e) => { + self.names.insert(e.cube_name().clone()); + } + MemberSymbolType::CubeTable(e) => { + self.names.insert(e.cube_name().clone()); + } + _ => {} + }; + Ok(Some(())) + } +} + +pub fn collect_cube_names(node: &Rc) -> Result, CubeError> { + let mut visitor = CubeNamesCollector::new(); + visitor.apply(node, &())?; + Ok(visitor.extract_result()) +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs index 98ee71eacb947..159cae3fe07f3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_cumulative_members.rs @@ -19,7 +19,12 @@ impl HasCumulativeMembersCollector { } impl TraversalVisitor for HasCumulativeMembersCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { match node.symbol() { MemberSymbolType::Measure(s) => { if s.is_rolling_window() { @@ -28,12 +33,16 @@ impl TraversalVisitor for HasCumulativeMembersCollector { } _ => {} }; - Ok(!self.has_cumulative_members) + if self.has_cumulative_members { + Ok(None) + } else { + Ok(Some(())) + } } } pub fn has_cumulative_members(node: &Rc) -> Result { let mut visitor = HasCumulativeMembersCollector::new(); - visitor.apply(node)?; + visitor.apply(node, &())?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs index f7b8b6234be21..98496c58a2af8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/has_multi_stage_members.rs @@ -21,7 +21,12 @@ impl HasMultiStageMembersCollector { } impl TraversalVisitor for HasMultiStageMembersCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { match node.symbol() { MemberSymbolType::Measure(s) => { if s.is_multi_stage() { @@ -32,10 +37,10 @@ impl TraversalVisitor for HasMultiStageMembersCollector { self.has_multi_stage = true; } else { for filter_node in s.measure_filters() { - self.apply(filter_node)? + self.apply(filter_node, &())? } for order_by in s.measure_order_by() { - self.apply(order_by.evaluation_node())? + self.apply(order_by.evaluation_node(), &())? } } } @@ -46,7 +51,11 @@ impl TraversalVisitor for HasMultiStageMembersCollector { } _ => {} }; - Ok(!self.has_multi_stage) + if self.has_multi_stage { + Ok(None) + } else { + Ok(Some(())) + } } } @@ -55,6 +64,6 @@ pub fn has_multi_stage_members( ignore_cumulative: bool, ) -> Result { let mut visitor = HasMultiStageMembersCollector::new(ignore_cumulative); - visitor.apply(node)?; + visitor.apply(node, &())?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs index 09e332098364e..456064bca38f5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/join_hints_collector.rs @@ -1,51 +1,73 @@ use crate::planner::sql_evaluator::{ EvaluationNode, MemberSymbol, MemberSymbolType, TraversalVisitor, }; +use crate::planner::BaseMeasure; use cubenativeutils::CubeError; use std::collections::HashSet; use std::rc::Rc; pub struct JoinHintsCollector { - hints: HashSet, + hints: Vec, } impl JoinHintsCollector { pub fn new() -> Self { - Self { - hints: HashSet::new(), - } + Self { hints: Vec::new() } } pub fn extract_result(self) -> Vec { - self.hints.into_iter().collect() + self.hints } } impl TraversalVisitor for JoinHintsCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { - let res = match node.symbol() { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { + match node.symbol() { MemberSymbolType::Dimension(e) => { if e.owned_by_cube() { - self.hints.insert(e.cube_name().clone()); + self.hints.push(e.cube_name().clone()); } - true } MemberSymbolType::Measure(e) => { + for filter_node in e.measure_filters() { + self.apply(filter_node, &())? + } + for order_by in e.measure_order_by() { + self.apply(order_by.evaluation_node(), &())? + } if e.owned_by_cube() { - self.hints.insert(e.cube_name().clone()); + self.hints.push(e.cube_name().clone()); } - true } MemberSymbolType::CubeName(e) => { - self.hints.insert(e.cube_name().clone()); - true + self.hints.push(e.cube_name().clone()); } MemberSymbolType::CubeTable(e) => { - self.hints.insert(e.cube_name().clone()); - true + self.hints.push(e.cube_name().clone()); } - _ => false, + _ => {} }; - Ok(res) + Ok(Some(())) + } +} + +pub fn collect_join_hints(node: &Rc) -> Result, CubeError> { + let mut visitor = JoinHintsCollector::new(); + visitor.apply(node, &())?; + Ok(visitor.extract_result()) +} + +pub fn collect_join_hints_for_measures( + measures: &Vec>, +) -> Result, CubeError> { + let mut visitor = JoinHintsCollector::new(); + for meas in measures.iter() { + visitor.apply(&meas.member_evaluator(), &())?; } + Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs index a2f4de39ddd23..fb7213e1494f3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/member_childs_collector.rs @@ -3,16 +3,23 @@ use cubenativeutils::CubeError; use std::rc::Rc; pub struct MemberChildsCollector { - pub is_root: bool, pub childs: Vec>, } +#[derive(Clone)] +pub struct MemberChildsCollectorState { + pub is_root: bool, +} + +impl MemberChildsCollectorState { + pub fn new(is_root: bool) -> Self { + Self { is_root } + } +} + impl MemberChildsCollector { pub fn new() -> Self { - Self { - is_root: true, - childs: vec![], - } + Self { childs: vec![] } } pub fn extract_result(self) -> Vec> { @@ -21,29 +28,34 @@ impl MemberChildsCollector { } impl TraversalVisitor for MemberChildsCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { - if self.is_root { - self.is_root = false; + type State = MemberChildsCollectorState; + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { + if state.is_root { + let new_state = MemberChildsCollectorState::new(false); match node.symbol() { MemberSymbolType::Measure(s) => { for filter_node in s.measure_filters() { - self.apply(filter_node)? + self.apply(filter_node, &new_state)? } for order_by in s.measure_order_by() { - self.apply(order_by.evaluation_node())? + self.apply(order_by.evaluation_node(), &new_state)? } - Ok(true) + Ok(Some(new_state)) } - MemberSymbolType::Dimension(_) => Ok(true), - _ => Ok(false), + MemberSymbolType::Dimension(_) => Ok(Some(new_state)), + _ => Ok(None), } } else { match node.symbol() { MemberSymbolType::Measure(_) | MemberSymbolType::Dimension(_) => { self.childs.push(node.clone()); - Ok(false) + Ok(None) } - _ => Ok(true), + _ => Ok(Some(state.clone())), } } } @@ -51,6 +63,6 @@ impl TraversalVisitor for MemberChildsCollector { pub fn member_childs(node: &Rc) -> Result>, CubeError> { let mut visitor = MemberChildsCollector::new(); - visitor.apply(node)?; + visitor.apply(node, &MemberChildsCollectorState::new(true))?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs index b291633ab01d5..1a3241bf740c4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/mod.rs @@ -1,11 +1,15 @@ +mod cube_names_collector; mod has_cumulative_members; mod has_multi_stage_members; mod join_hints_collector; mod member_childs_collector; mod multiplied_measures_collector; +pub use cube_names_collector::collect_cube_names; pub use has_cumulative_members::{has_cumulative_members, HasCumulativeMembersCollector}; pub use has_multi_stage_members::{has_multi_stage_members, HasMultiStageMembersCollector}; -pub use join_hints_collector::JoinHintsCollector; +pub use join_hints_collector::{ + collect_join_hints, collect_join_hints_for_measures, JoinHintsCollector, +}; pub use member_childs_collector::{member_childs, MemberChildsCollector}; pub use multiplied_measures_collector::{collect_multiplied_measures, MultipliedMeasuresCollector}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs index 8ee83c026ec8e..cf638e8b3a024 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/collectors/multiplied_measures_collector.rs @@ -2,36 +2,94 @@ use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::{ EvaluationNode, MemberSymbol, MemberSymbolType, TraversalVisitor, }; +use crate::planner::BaseMeasure; use cubenativeutils::CubeError; +use std::collections::HashSet; use std::rc::Rc; -pub struct RootMeasureResult { +struct CompositeMeasuresCollector { + composite_measures: HashSet, +} + +struct CompositeMeasureCollectorState { + pub parent_measure: Option>, +} + +impl CompositeMeasureCollectorState { + pub fn new(parent_measure: Option>) -> Self { + Self { parent_measure } + } +} + +impl CompositeMeasuresCollector { + pub fn new() -> Self { + Self { + composite_measures: HashSet::new(), + } + } + + pub fn extract_result(self) -> HashSet { + self.composite_measures + } +} + +impl TraversalVisitor for CompositeMeasuresCollector { + type State = CompositeMeasureCollectorState; + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { + let res = match node.symbol() { + MemberSymbolType::Measure(e) => { + if let Some(parent) = &state.parent_measure { + if parent.cube_name() != node.cube_name() { + self.composite_measures.insert(parent.full_name()); + } + } + + let new_state = CompositeMeasureCollectorState::new(Some(node.clone())); + Some(new_state) + } + MemberSymbolType::Dimension(_) => None, + _ => None, + }; + Ok(res) + } +} + +pub struct MeasureResult { pub multiplied: bool, - pub measure: String, + pub measure: Rc, } pub struct MultipliedMeasuresCollector { query_tools: Rc, - parent_measure: Option, - root_measure: Option, + composite_measures: HashSet, + colllected_measures: Vec, } impl MultipliedMeasuresCollector { - pub fn new(query_tools: Rc) -> Self { + pub fn new(query_tools: Rc, composite_measures: HashSet) -> Self { Self { query_tools, - parent_measure: None, - root_measure: None, + composite_measures, + colllected_measures: vec![], } } - pub fn extract_result(self) -> Option { - self.root_measure + pub fn extract_result(self) -> Vec { + self.colllected_measures } } impl TraversalVisitor for MultipliedMeasuresCollector { - fn on_node_traverse(&mut self, node: &Rc) -> Result { + type State = (); + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError> { let res = match node.symbol() { MemberSymbolType::Measure(e) => { let full_name = e.full_name(); @@ -43,17 +101,22 @@ impl TraversalVisitor for MultipliedMeasuresCollector { .unwrap_or(&false) .clone(); - if self.parent_measure.is_none() { - self.root_measure = Some(RootMeasureResult { + if !self.composite_measures.contains(&full_name) { + self.colllected_measures.push(MeasureResult { multiplied, - measure: full_name.clone(), + measure: BaseMeasure::try_new(node.clone(), self.query_tools.clone())? + .unwrap(), }) } - self.parent_measure = Some(full_name); - true + + if self.composite_measures.contains(&full_name) { + Some(()) + } else { + None + } } - MemberSymbolType::Dimension(_) => true, - _ => false, + MemberSymbolType::Dimension(_) => None, + _ => None, }; Ok(res) } @@ -62,8 +125,11 @@ impl TraversalVisitor for MultipliedMeasuresCollector { pub fn collect_multiplied_measures( query_tools: Rc, node: &Rc, -) -> Result, CubeError> { - let mut visitor = MultipliedMeasuresCollector::new(query_tools); - visitor.apply(node)?; +) -> Result, CubeError> { + let mut composite_collector = CompositeMeasuresCollector::new(); + composite_collector.apply(node, &CompositeMeasureCollectorState::new(None))?; + let composite_measures = composite_collector.extract_result(); + let mut visitor = MultipliedMeasuresCollector::new(query_tools, composite_measures); + visitor.apply(node, &())?; Ok(visitor.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs index 97c9eac59b4e3..f656c7bafe945 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/compiler.rs @@ -115,7 +115,7 @@ impl Compiler { pub fn join_hints(&self) -> Result, CubeError> { let mut collector = JoinHintsCollector::new(); for member in self.members.values() { - collector.apply(member)?; + collector.apply(member, &())?; } Ok(collector.extract_result()) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs index 5c53c7833ca61..70ebbf91d2ad1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/evaluation_node.rs @@ -66,6 +66,10 @@ impl EvaluationNode { self.symbol.name() } + pub fn cube_name(&self) -> String { + self.symbol.cube_name() + } + pub fn is_measure(&self) -> bool { self.symbol.is_measure() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs index a8a63d4f21b55..e77d5489c485c 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_node_transformers/set_schema.rs @@ -3,7 +3,7 @@ use crate::planner::sql_evaluator::sql_nodes::final_measure::FinalMeasureSqlNode use crate::planner::sql_evaluator::sql_nodes::{ AutoPrefixSqlNode, EvaluateSqlNode, MeasureFilterSqlNode, MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RollingWindowNode, RootSqlNode, SqlNode, - TimeShiftSqlNode, + TimeShiftSqlNode, UngroupedMeasureSqlNode, UngroupedQueryFinalMeasureSqlNode, }; use std::rc::Rc; @@ -88,6 +88,20 @@ pub fn set_schema_impl(sql_node: Rc, schema: Rc) -> Rc() { let input = set_schema_impl(time_shift.input().clone(), schema.clone()); TimeShiftSqlNode::new(time_shift.shifts().clone(), input) + } else if let Some(ungrouped_measure) = sql_node + .clone() + .as_any() + .downcast_ref::() + { + let input = set_schema_impl(ungrouped_measure.input().clone(), schema.clone()); + UngroupedMeasureSqlNode::new(input) + } else if let Some(ungrouped_measure) = sql_node + .clone() + .as_any() + .downcast_ref::() + { + let input = set_schema_impl(ungrouped_measure.input().clone(), schema.clone()); + UngroupedQueryFinalMeasureSqlNode::new(input) } else { unreachable!("Not all nodes are implemented in set_schema function"); } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs index 0acd90c5f868d..2550fda974ee4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs @@ -1,4 +1,5 @@ use super::SqlNode; +use crate::cube_bridge::memeber_sql::MemberSqlArg; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs index 3368a23781deb..48ead328a90c7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/factory.rs @@ -1,29 +1,47 @@ use super::{ AutoPrefixSqlNode, EvaluateSqlNode, FinalMeasureSqlNode, MeasureFilterSqlNode, MultiStageRankNode, MultiStageWindowNode, RenderReferencesSqlNode, RollingWindowNode, - RootSqlNode, SqlNode, TimeShiftSqlNode, + RootSqlNode, SqlNode, TimeShiftSqlNode, UngroupedMeasureSqlNode, + UngroupedQueryFinalMeasureSqlNode, }; use std::collections::HashMap; use std::rc::Rc; +#[derive(Clone)] pub struct SqlNodesFactory { time_shifts: Option>, + ungrouped: bool, } impl SqlNodesFactory { pub fn new() -> Rc { - Rc::new(Self { time_shifts: None }) + Rc::new(Self { + time_shifts: None, + ungrouped: false, + }) + } + pub fn new_ungroupped() -> Rc { + Rc::new(Self { + time_shifts: None, + ungrouped: true, + }) } pub fn new_with_time_shifts(time_shifts: HashMap) -> Rc { Rc::new(Self { time_shifts: Some(time_shifts), + ungrouped: false, }) } + pub fn default_node_processor(&self) -> Rc { let evaluate_sql_processor = EvaluateSqlNode::new(); let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); let measure_filter_processor = MeasureFilterSqlNode::new(auto_prefix_processor.clone()); - let final_measure_processor = FinalMeasureSqlNode::new(measure_filter_processor.clone()); + let final_measure_processor: Rc = if self.ungrouped { + UngroupedQueryFinalMeasureSqlNode::new(measure_filter_processor.clone()) + } else { + FinalMeasureSqlNode::new(measure_filter_processor.clone()) + }; let root_node = RootSqlNode::new( self.dimension_processor(auto_prefix_processor.clone()), final_measure_processor.clone(), @@ -33,6 +51,21 @@ impl SqlNodesFactory { RenderReferencesSqlNode::new(root_node) } + pub fn ungroupped_measure_node_processor(&self) -> Rc { + let evaluate_sql_processor = EvaluateSqlNode::new(); + let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); + let measure_filter_processor = MeasureFilterSqlNode::new(auto_prefix_processor.clone()); + let ungrouped_measure_processor = + UngroupedMeasureSqlNode::new(measure_filter_processor.clone()); + let root_node = RootSqlNode::new( + self.dimension_processor(auto_prefix_processor.clone()), + ungrouped_measure_processor.clone(), + auto_prefix_processor.clone(), + evaluate_sql_processor.clone(), + ); + RenderReferencesSqlNode::new(root_node) + } + pub fn multi_stage_rank_node_processor(&self, partition: Vec) -> Rc { let evaluate_sql_processor = EvaluateSqlNode::new(); let auto_prefix_processor = AutoPrefixSqlNode::new(evaluate_sql_processor.clone()); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs index c5469d5206242..7117e4815a7d7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/final_measure.rs @@ -1,4 +1,5 @@ use super::SqlNode; +use crate::cube_bridge::memeber_sql::MemberSqlArg; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; @@ -30,12 +31,22 @@ impl SqlNode for FinalMeasureSqlNode { ) -> Result { let res = match node.symbol() { MemberSymbolType::Measure(ev) => { - let input = self.input.to_sql( - visitor, - node, - query_tools.clone(), - node_processor.clone(), - )?; + let input = if ev.is_splitted_source() { + let args = visitor.evaluate_deps(node, node_processor.clone())?; + //FIXME hack for working with + //measures like rolling window + if !args.is_empty() { + match &args[0] { + MemberSqlArg::String(s) => s.clone(), + _ => "".to_string(), + } + } else { + "".to_string() + } + } else { + self.input + .to_sql(visitor, node, query_tools.clone(), node_processor.clone())? + }; if ev.is_calculated() { input @@ -45,6 +56,7 @@ impl SqlNode for FinalMeasureSqlNode { } else { &ev.measure_type() }; + format!("{}({})", measure_type, input) } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs index f316c5d8facb3..60a1c025855c5 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/mod.rs @@ -10,6 +10,8 @@ pub mod rolling_window; pub mod root_processor; pub mod sql_node; pub mod time_shift; +pub mod ungroupped_measure; +pub mod ungroupped_query_final_measure; pub use auto_prefix::AutoPrefixSqlNode; pub use evaluate_sql::EvaluateSqlNode; @@ -23,3 +25,5 @@ pub use rolling_window::RollingWindowNode; pub use root_processor::RootSqlNode; pub use sql_node::SqlNode; pub use time_shift::TimeShiftSqlNode; +pub use ungroupped_measure::UngroupedMeasureSqlNode; +pub use ungroupped_query_final_measure::UngroupedQueryFinalMeasureSqlNode; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_measure.rs new file mode 100644 index 0000000000000..926113d92cefb --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_measure.rs @@ -0,0 +1,62 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct UngroupedMeasureSqlNode { + input: Rc, +} + +impl UngroupedMeasureSqlNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) + } + + pub fn input(&self) -> &Rc { + &self.input + } +} + +impl SqlNode for UngroupedMeasureSqlNode { + fn to_sql( + &self, + visitor: &mut SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let res = match node.symbol() { + MemberSymbolType::Measure(ev) => { + let input = self.input.to_sql( + visitor, + node, + query_tools.clone(), + node_processor.clone(), + )?; + + if input == "*" { + "1".to_string() + } else { + input + } + } + _ => { + return Err(CubeError::internal(format!( + "Measure filter node processor called for wrong node", + ))); + } + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_query_final_measure.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_query_final_measure.rs new file mode 100644 index 0000000000000..5a2044c30270d --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/ungroupped_query_final_measure.rs @@ -0,0 +1,69 @@ +use super::SqlNode; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::SqlEvaluatorVisitor; +use crate::planner::sql_evaluator::{EvaluationNode, MemberSymbolType}; +use cubenativeutils::CubeError; +use std::any::Any; +use std::rc::Rc; + +pub struct UngroupedQueryFinalMeasureSqlNode { + input: Rc, +} + +impl UngroupedQueryFinalMeasureSqlNode { + pub fn new(input: Rc) -> Rc { + Rc::new(Self { input }) + } + + pub fn input(&self) -> &Rc { + &self.input + } +} + +impl SqlNode for UngroupedQueryFinalMeasureSqlNode { + fn to_sql( + &self, + visitor: &mut SqlEvaluatorVisitor, + node: &Rc, + query_tools: Rc, + node_processor: Rc, + ) -> Result { + let res = match node.symbol() { + MemberSymbolType::Measure(ev) => { + let input = self.input.to_sql( + visitor, + node, + query_tools.clone(), + node_processor.clone(), + )?; + + if input == "*" { + "1".to_string() + } else { + if ev.measure_type() == "count" + || ev.measure_type() == "countDistinct" + || ev.measure_type() == "countDistinctApprox" + { + format!("CASE WHEN ({}) IS NOT NULL THEN 1 END", input) //TODO templates!! + } else { + input + } + } + } + _ => { + return Err(CubeError::internal(format!( + "Measure filter node processor called for wrong node", + ))); + } + }; + Ok(res) + } + + fn as_any(self: Rc) -> Rc { + self.clone() + } + + fn childs(&self) -> Vec> { + vec![self.input.clone()] + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs index d719fb93492c2..4beabaa483ae1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol_type.rs @@ -28,6 +28,16 @@ impl MemberSymbolType { MemberSymbolType::SimpleSql(_) => "".to_string(), } } + + pub fn cube_name(&self) -> String { + match self { + MemberSymbolType::Dimension(d) => d.cube_name().clone(), + MemberSymbolType::Measure(m) => m.cube_name().clone(), + MemberSymbolType::CubeName(c) => c.cube_name().clone(), + MemberSymbolType::CubeTable(c) => c.cube_name().clone(), + MemberSymbolType::SimpleSql(_) => "".to_string(), + } + } pub fn is_measure(&self) -> bool { matches!(self, Self::Measure(_)) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs index 0a9a49b14f681..bf0192ce4ea64 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/visitor.rs @@ -3,18 +3,27 @@ use cubenativeutils::CubeError; use std::rc::Rc; pub trait TraversalVisitor { - fn on_node_traverse(&mut self, node: &Rc) -> Result; + type State; + fn on_node_traverse( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result, CubeError>; - fn apply(&mut self, node: &Rc) -> Result<(), CubeError> { - if self.on_node_traverse(node)? { - self.travese_deps(node)?; + fn apply(&mut self, node: &Rc, state: &Self::State) -> Result<(), CubeError> { + if let Some(state) = self.on_node_traverse(node, state)? { + self.travese_deps(node, &state)?; } Ok(()) } - fn travese_deps(&mut self, node: &Rc) -> Result<(), CubeError> { + fn travese_deps( + &mut self, + node: &Rc, + state: &Self::State, + ) -> Result<(), CubeError> { for dep in node.deps() { - self.traverse_single_dep(dep, node)?; + self.traverse_single_dep(dep, node, state)?; } Ok(()) } @@ -23,20 +32,21 @@ pub trait TraversalVisitor { &mut self, dep: &Dependency, node: &Rc, + state: &Self::State, ) -> Result<(), CubeError> { match dep { - Dependency::SingleDependency(dep) => self.apply(dep), + Dependency::SingleDependency(dep) => self.apply(dep, state), Dependency::StructDependency(dep) => { if dep.sql_fn.is_some() { - self.apply(node)?; + self.apply(node, state)?; } if let Some(to_string_fn) = &dep.to_string_fn { - self.apply(to_string_fn)?; + self.apply(to_string_fn, state)?; } for (_, v) in dep.properties.iter() { match v { Dependency::SingleDependency(dep) => { - self.apply(dep)?; + self.apply(dep, state)?; } Dependency::StructDependency(_) => unimplemented!(), Dependency::ContextDependency(_) => {} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs index c7eaaf9a8ed11..05f9a85af6680 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/filter.rs @@ -1,7 +1,7 @@ use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use cubenativeutils::CubeError; use minijinja::context; -use std::rc::Rc; +use std::{ptr::null, rc::Rc}; #[derive(Clone)] pub struct FilterTemplates { @@ -182,11 +182,37 @@ impl FilterTemplates { ) } - fn additional_null_check(&self, need: bool, column: &String) -> Result { + pub fn additional_null_check(&self, need: bool, column: &String) -> Result { if need { self.or_is_null_check(column.clone()) } else { Ok(String::default()) } } + + pub fn ilike( + &self, + column: &str, + value: &str, + start_wild: bool, + end_wild: bool, + not: bool, + ) -> Result { + let pattern = self.render.render_template( + &"filters/like_pattern", + context! { + start_wild => start_wild, + value => value, + end_wild => end_wild + }, + )?; + self.render.render_template( + &"expressions/ilike", + context! { + expr => column.clone(), + negated => not, + pattern => pattern + }, + ) + } }