diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts b/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts index 9dd2bc613a8a4..49f17519739aa 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts @@ -1,5 +1,3 @@ -import R from 'ramda'; - export class BaseGroupFilter { protected readonly values: any; @@ -31,7 +29,7 @@ export class BaseGroupFilter { return null; } return `(${sql})`; - }).filter(R.identity).join(` ${this.operator.toUpperCase()} `); + }).filter(x => x).join(` ${this.operator.toUpperCase()} `); if (!r.length) { return null; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 43ded98bf2b62..9f1a73f4b1bb7 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -2159,15 +2159,16 @@ export class BaseQuery { const memberPathArray = [cubeName, name]; const memberPath = this.cubeEvaluator.pathFromArray(memberPathArray); let type = memberExpressionType; - if (!type && this.cubeEvaluator.isMeasure(memberPathArray)) { - type = 'measure'; - } - if (!type && this.cubeEvaluator.isDimension(memberPathArray)) { - type = 'dimension'; - } - if (!type && this.cubeEvaluator.isSegment(memberPathArray)) { - type = 'segment'; + if (!type) { + if (this.cubeEvaluator.isMeasure(memberPathArray)) { + type = 'measure'; + } else if (this.cubeEvaluator.isDimension(memberPathArray)) { + type = 'dimension'; + } else if (this.cubeEvaluator.isSegment(memberPathArray)) { + type = 'segment'; + } } + const parentMember = this.safeEvaluateSymbolContext().currentMember; if (this.safeEvaluateSymbolContext().memberChildren && parentMember) { this.safeEvaluateSymbolContext().memberChildren[parentMember] = this.safeEvaluateSymbolContext().memberChildren[parentMember] || []; @@ -2336,7 +2337,7 @@ export class BaseQuery { /** * Evaluate escaped SQL-alias for cube or cube's property - * (measure, dimention). + * (measure, dimension). * @param {string} cubeName * @returns string */ @@ -3494,25 +3495,29 @@ export class BaseQuery { static extractFilterMembers(filter) { if (filter.operator === 'and' || filter.operator === 'or') { return filter.values.map(f => BaseQuery.extractFilterMembers(f)).reduce((a, b) => ((a && b) ? { ...a, ...b } : null), {}); - } else if (filter.measure || filter.dimension) { + } else if (filter.measure) { + return { + [filter.measure]: true + }; + } else if (filter.dimension) { return { - [filter.measure || filter.dimension]: true + [filter.dimension]: true }; } else { return null; } } - static findAndSubTreeForFilterGroup(filter, groupMembers, newGroupFilter) { + static findAndSubTreeForFilterGroup(filter, groupMembers, newGroupFilter, aliases) { if ((filter.operator === 'and' || filter.operator === 'or') && !filter.values?.length) { return null; } const filterMembers = BaseQuery.extractFilterMembers(filter); - if (filterMembers && Object.keys(filterMembers).every(m => groupMembers.indexOf(m) !== -1)) { + if (filterMembers && Object.keys(filterMembers).every(m => (groupMembers.indexOf(m) !== -1 || aliases.indexOf(m) !== -1))) { return filter; } if (filter.operator === 'and') { - const result = filter.values.map(f => BaseQuery.findAndSubTreeForFilterGroup(f, groupMembers, newGroupFilter)).filter(f => !!f); + const result = filter.values.map(f => BaseQuery.findAndSubTreeForFilterGroup(f, groupMembers, newGroupFilter, aliases)).filter(f => !!f); if (!result.length) { return null; } @@ -3537,21 +3542,30 @@ export class BaseQuery { ); } - static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter) { + static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases) { if (!filter) { return '1 = 1'; } if (filter.operator === 'and' || filter.operator === 'or') { const values = filter.values - .map(f => BaseQuery.renderFilterParams(f, filterParamArgs, allocateParam, newGroupFilter)) + .map(f => BaseQuery.renderFilterParams(f, filterParamArgs, allocateParam, newGroupFilter, aliases)) .map(v => ({ filterToWhere: () => v })); return newGroupFilter({ operator: filter.operator, values }).filterToWhere(); } - const filterParams = filter && filter.filterParams(); - const filterParamArg = filterParamArgs.filter(p => p.__member() === filter.measure || p.__member() === filter.dimension)[0]; + const filterParams = filter.filterParams(); + const filterParamArg = filterParamArgs.filter(p => { + const member = p.__member(); + return member === filter.measure || + member === filter.dimension || + (aliases[member] && ( + aliases[member] === filter.measure || + aliases[member] === filter.dimension + )); + })[0]; + if (!filterParamArg) { throw new Error(`FILTER_PARAMS arg not found for ${filter.measure || filter.dimension}`); } @@ -3584,15 +3598,25 @@ export class BaseQuery { return f.__member(); }); - const filter = BaseQuery.findAndSubTreeForFilterGroup(newGroupFilter({ operator: 'and', values: allFilters }), groupMembers, newGroupFilter); + const aliases = allFilters ? + allFilters + .map(v => (v.query ? v.query.allBackAliasMembersExceptSegments() : {})) + .reduce((a, b) => ({ ...a, ...b }), {}) + : {}; + const filter = BaseQuery.findAndSubTreeForFilterGroup( + newGroupFilter({ operator: 'and', values: allFilters }), + groupMembers, + newGroupFilter, + Object.values(aliases) + ); - return `(${BaseQuery.renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter)})`; + return `(${BaseQuery.renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases)})`; }; } static filterProxyFromAllFilters(allFilters, cubeEvaluator, allocateParam, newGroupFilter) { return new Proxy({}, { - get: (target, name) => { + get: (_target, name) => { if (name === '_objectWithResolvedProperties') { return true; } @@ -3609,12 +3633,28 @@ export class BaseQuery { return cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName]); }, toString() { + // Segments should be excluded because they are evaluated separately in cubeReferenceProxy + // In other case this falls into the recursive loop/stack exceeded caused by: + // collectFrom() -> traverseSymbol() -> evaluateSymbolSql() -> + // evaluateSql() -> resolveSymbolsCall() -> cubeReferenceProxy->toString() -> + // evaluateSymbolSql() -> evaluateSql()... -> and got here again + const aliases = allFilters ? + allFilters + .map(v => (v.query ? v.query.allBackAliasMembersExceptSegments() : {})) + .reduce((a, b) => ({ ...a, ...b }), {}) + : {}; + // Filtering aliases that somehow relate to this cube + const filteredAliases = Object.entries(aliases) + .filter(([key, value]) => key.startsWith(cubeNameObj.cube) || value.startsWith(cubeNameObj.cube)) + .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {}); const filter = BaseQuery.findAndSubTreeForFilterGroup( newGroupFilter({ operator: 'and', values: allFilters }), [cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName])], - newGroupFilter + newGroupFilter, + Object.values(filteredAliases) ); - return `(${BaseQuery.renderFilterParams(filter, [this], allocateParam, newGroupFilter)})`; + + return `(${BaseQuery.renderFilterParams(filter, [this], allocateParam, newGroupFilter, aliases)})`; } }) }) @@ -3622,4 +3662,46 @@ export class BaseQuery { } }); } + + flattenAllMembers(excludeSegments = false) { + return R.flatten( + this.measures + .concat(this.dimensions) + .concat(excludeSegments ? [] : this.segments) + .concat(this.filters) + .concat(this.measureFilters) + .concat(this.timeDimensions) + .map(m => m.getMembers()), + ); + } + + allBackAliasMembersExceptSegments() { + return this.backAliasMembers(this.flattenAllMembers(true)); + } + + allBackAliasMembers() { + return this.backAliasMembers(this.flattenAllMembers()); + } + + backAliasMembers(members) { + const query = this; + return members.map( + member => { + const collectedMembers = query + .collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor'); + const memberPath = member.expressionPath(); + let nonAliasSeen = false; + return collectedMembers + .filter(d => { + if (!query.cubeEvaluator.byPathAnyType(d).aliasMember) { + nonAliasSeen = true; + } + return !nonAliasSeen; + }) + .map(d => ( + { [query.cubeEvaluator.byPathAnyType(d).aliasMember]: memberPath } + )).reduce((a, b) => ({ ...a, ...b }), {}); + } + ).reduce((a, b) => ({ ...a, ...b }), {}); + } } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index bded2c1e0365d..9edc16c70ca53 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -157,7 +157,7 @@ export class PreAggregations { const queryForSqlEvaluation = this.query.preAggregationQueryForSqlEvaluation(cube, preAggregation); const partitionInvalidateKeyQueries = queryForSqlEvaluation.partitionInvalidateKeyQueries && queryForSqlEvaluation.partitionInvalidateKeyQueries(cube, preAggregation); - const allBackAliasMembers = PreAggregations.allBackAliasMembers(this.query); + const allBackAliasMembers = this.query.allBackAliasMembers(); const matchedTimeDimension = preAggregation.partitionGranularity && !this.hasCumulativeMeasures && this.query.timeDimensions.find(td => { @@ -292,7 +292,7 @@ export class PreAggregations { static transformQueryToCanUseForm(query) { const flattenDimensionMembers = this.flattenDimensionMembers(query); const sortedDimensions = this.squashDimensions(query, flattenDimensionMembers); - const allBackAliasMembers = this.allBackAliasMembers(query); + const allBackAliasMembers = query.allBackAliasMembers(); const measures = query.measures.concat(query.measureFilters); const measurePaths = R.uniq(this.flattenMembers(measures).map(m => m.expressionPath())); const collectLeafMeasures = query.collectLeafMeasures.bind(query); @@ -426,31 +426,6 @@ export class PreAggregations { ); } - static backAliasMembers(query, members) { - return members.map( - member => { - const collectedMembers = query - .collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor'); - const memberPath = member.expressionPath(); - let nonAliasSeen = false; - return collectedMembers - .filter(d => { - if (!query.cubeEvaluator.byPathAnyType(d).aliasMember) { - nonAliasSeen = true; - } - return !nonAliasSeen; - }) - .map(d => ( - { [query.cubeEvaluator.byPathAnyType(d).aliasMember]: memberPath } - )).reduce((a, b) => ({ ...a, ...b }), {}); - } - ).reduce((a, b) => ({ ...a, ...b }), {}); - } - - static allBackAliasMembers(query) { - return this.backAliasMembers(query, this.flattenAllMembers(query)); - } - static sortTimeDimensionsWithRollupGranularity(timeDimensions) { return timeDimensions && R.sortBy( R.prop(0), @@ -750,18 +725,6 @@ export class PreAggregations { ); } - static flattenAllMembers(query) { - return R.flatten( - query.measures - .concat(query.dimensions) - .concat(query.segments) - .concat(query.filters) - .concat(query.measureFilters) - .concat(query.timeDimensions) - .map(m => m.getMembers()), - ); - } - // eslint-disable-next-line no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars getCubeLattice(cube, preAggregationName, preAggregation) { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts index 74fd630b1379f..f3d67dc166574 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts @@ -10,7 +10,7 @@ describe('YAMLCompiler', () => { cubes: - name: ActiveUsers sql: "SELECT 1 as user_id, '2022-01-01' as timestamp" - + measures: - name: weeklyActive sql: "{CUBE}.user_id" @@ -62,7 +62,7 @@ cubes: cubes: - name: ActiveUsers sql: "SELECT 1 as user_id, '2022-01-01' as timestamp" - + measures: - name: weeklyActive sql: "{CUBE}.user_id" @@ -83,7 +83,7 @@ cubes: cubes: - name: ActiveUsers sql: "SELECT 1 as user_id, '2022-01-01'::timestamptz as timestamp" - + measures: - name: withFilter sql: "{CUBE}.user_id" @@ -126,7 +126,7 @@ cubes: cubes: - name: ActiveUsers sql: "SELECT 1 as user_id, '2022-01-01' as timestamp" - + measures: - name: weeklyActive sql: "{user_id}" @@ -181,7 +181,7 @@ cubes: cubes: - name: ActiveUsers sql: "SELECT 1 as user_id, '2022-01-01' as timestamp" - + measures: - name: weeklyActive sql: "{CUBE.user_id}" @@ -197,7 +197,7 @@ cubes: - name: time sql: "{CUBE}.timestamp" type: time - + preAggregations: - name: main measures: @@ -248,7 +248,7 @@ cubes: cubes: - name: active_users sql: "SELECT * FROM (SELECT 1 as user_id, '2022-01-01'::timestamptz as \\"timestamp\\") t WHERE {FILTER_PARAMS.active_users.time.filter(\\"timestamp\\")} AND {FILTER_PARAMS.active_users.time.filter(lambda a,b : f'timestamp >= {a}::timestamptz AND timestamp <= {b}::timestamptz')}" - + measures: - name: weekly_active sql: "{CUBE.user_id}" @@ -303,13 +303,20 @@ cubes: const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(` cubes: - name: orders - sql: "SELECT 1 as id, 1 as customer_id, TO_TIMESTAMP('2022-01-01', 'YYYY-MM-DD') as timestamp WHERE {FILTER_PARAMS.orders.time.filter(\\"timestamp\\")}" - + sql: "SELECT * + FROM ( + SELECT + 1 as id, + 1 as customer_id, + TO_TIMESTAMP('2022-01-01', 'YYYY-MM-DD') as timestamp + ) + WHERE {FILTER_PARAMS.orders.time.filter(\\"timestamp\\")}" + joins: - name: customers sql: "{CUBE}.customer_id = {customers}.id" relationship: many_to_one - + measures: - name: count type: count @@ -319,11 +326,11 @@ cubes: sql: "{CUBE}.id" type: string primary_key: true - + - name: time sql: "{CUBE}.timestamp" type: time - + preAggregations: - name: main measures: [orders.count] @@ -356,11 +363,11 @@ cubes: measures: - name: count type: count - - + + - name: customers sql: "SELECT 1 as id, 'Foo' as name" - + measures: - name: count type: count @@ -370,11 +377,11 @@ cubes: sql: id type: string primary_key: true - + - name: name sql: "{CUBE}.name" type: string - + views: - name: line_items_view @@ -385,13 +392,13 @@ views: - join_path: line_items.orders prefix: true includes: "*" - excludes: + excludes: - count - + - join_path: line_items.orders.customers alias: aliased_customers prefix: true - includes: + includes: - name: name alias: full_name `); @@ -425,12 +432,12 @@ views: cubes: - name: BaseUsers sql: "SELECT 1" - + dimensions: - name: time sql: "{CUBE}.timestamp" type: time - + - name: ActiveUsers sql: "SELECT 1 as user_id, '2022-01-01' as timestamp" extends: BaseUsers @@ -527,9 +534,9 @@ cubes: type: string sql: w_id primary_key: true - + joins: - + - name: Z sql: "{CUBE}.z_id = {Z}.z_id" relationship: many_to_one @@ -550,9 +557,9 @@ cubes: type: string sql: m_id primary_key: true - + joins: - + - name: V sql: "{CUBE}.v_id = {V}.v_id" relationship: many_to_one @@ -560,11 +567,11 @@ cubes: - name: W sql: "{CUBE}.w_id = {W}.w_id" relationship: many_to_one - + - name: Z sql: > SELECT 1 as z_id, 'US' as COUNTRY - + dimensions: - name: country sql: "{CUBE}.COUNTRY" @@ -574,7 +581,7 @@ cubes: sql: "{CUBE}.z_id" type: string primaryKey: true - + - name: V sql: | SELECT 1 as v_id, 1 as z_id @@ -595,7 +602,7 @@ cubes: views: - name: m_view - + cubes: - join_path: M diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index 3a179545cd28e..a0b1fa3e9f535 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -660,21 +660,30 @@ describe('SQL Generation', () => { /** @type {Compilers} */ const compilers = prepareYamlCompiler( createSchemaYaml({ - cubes: [ - { - name: 'Order', - sql: 'select * from order where {FILTER_PARAMS.Order.type.filter(\'type\')}', - measures: [{ - name: 'count', - type: 'count', - }], - dimensions: [{ - name: 'type', - sql: 'type', - type: 'string' - }] - }, - ] + cubes: [{ + name: 'Order', + sql: 'select * from order where {FILTER_PARAMS.Order.type.filter(\'type\')}', + measures: [{ + name: 'count', + type: 'count', + }], + dimensions: [{ + name: 'type', + sql: 'type', + type: 'string' + }] + }], + views: [{ + name: 'orders_view', + cubes: [{ + join_path: 'Order', + prefix: true, + includes: [ + 'type', + 'count', + ] + }] + }] }) ); @@ -829,6 +838,23 @@ describe('SQL Generation', () => { const cubeSQL = query.cubeSql('Order'); expect(cubeSQL).toMatch(/\(\s*\(.*type\s*=\s*\$\d\$.*OR.*type\s*=\s*\$\d\$.*\)\s*AND\s*\(.*type\s*=\s*\$\d\$.*OR.*type\s*=\s*\$\d\$.*\)\s*\)/); }); + + it('propagate filter params from view into cube\'s query', async () => { + await compilers.compiler.compile(); + const query = new BaseQuery(compilers, { + measures: ['orders_view.Order_count'], + filters: [ + { + member: 'orders_view.Order_type', + operator: 'equals', + values: ['online'], + }, + ], + }); + const cubeSQL = query.cubeSql('Order'); + console.log('TEST: ', cubeSQL); + expect(cubeSQL).toContain('select * from order where ((type = $0$))'); + }); }); }); @@ -866,13 +892,13 @@ describe('Class unit tests', () => { expect(baseQuery.aliasName('CamelCaseCube.id', false)).toEqual('camel_case_cube__id'); expect(baseQuery.aliasName('CamelCaseCube.description', false)).toEqual('camel_case_cube__description'); expect(baseQuery.aliasName('CamelCaseCube.grant_total', false)).toEqual('camel_case_cube__grant_total'); - + // aliasName for pre-agg expect(baseQuery.aliasName('CamelCaseCube', true)).toEqual('camel_case_cube'); expect(baseQuery.aliasName('CamelCaseCube.id', true)).toEqual('camel_case_cube_id'); expect(baseQuery.aliasName('CamelCaseCube.description', true)).toEqual('camel_case_cube_description'); expect(baseQuery.aliasName('CamelCaseCube.grant_total', true)).toEqual('camel_case_cube_grant_total'); - + // cubeAlias expect(baseQuery.cubeAlias('CamelCaseCube')).toEqual('"camel_case_cube"'); expect(baseQuery.cubeAlias('CamelCaseCube.id')).toEqual('"camel_case_cube__id"'); @@ -914,7 +940,7 @@ describe('Class unit tests', () => { expect(baseQuery.aliasName('CamelCaseCube.id', false)).toEqual('t1__id'); expect(baseQuery.aliasName('CamelCaseCube.description', false)).toEqual('t1__description'); expect(baseQuery.aliasName('CamelCaseCube.grant_total', false)).toEqual('t1__grant_total'); - + // aliasName for pre-agg expect(baseQuery.aliasName('CamelCaseCube', true)).toEqual('t1'); expect(baseQuery.aliasName('CamelCaseCube.id', true)).toEqual('t1_id'); diff --git a/packages/cubejs-server-core/src/core/CompilerApi.js b/packages/cubejs-server-core/src/core/CompilerApi.js index 489c1816d6cb5..401e50ac5d46c 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.js +++ b/packages/cubejs-server-core/src/core/CompilerApi.js @@ -129,7 +129,7 @@ export class CompilerApi { async getSqlGenerator(query, dataSource) { const dbType = await this.getDbType(dataSource); const compilers = await this.getCompilers({ requestId: query.requestId }); - let sqlGenerator = await this.createQueryByDataSource(compilers, query, dataSource); + let sqlGenerator = await this.createQueryByDataSource(compilers, query, dataSource, dbType); if (!sqlGenerator) { throw new Error(`Unknown dbType: ${dbType}`); @@ -142,7 +142,8 @@ export class CompilerApi { sqlGenerator = await this.createQueryByDataSource( compilers, query, - dataSource + dataSource, + _dbType ); if (!sqlGenerator) { @@ -203,8 +204,10 @@ export class CompilerApi { return cubeEvaluator.scheduledPreAggregations(); } - async createQueryByDataSource(compilers, query, dataSource) { - const dbType = await this.getDbType(dataSource); + async createQueryByDataSource(compilers, query, dataSource, dbType) { + if (!dbType) { + dbType = await this.getDbType(dataSource); + } return this.createQuery(compilers, dbType, this.getDialectClass(dataSource, dbType), query); }