fix(schema-compiler): propagate FILTER_PARAMS from view to inner cube's SELECT (#8466)

* chore(schema-compiler): Move backAliasMembers*() from PreAggreggations to BaseQuery

* chore(schema-compiler): remove Ramda from BaseGroupFilter

* chore(schema-compiler): improve getSqlGenerator() by removing duplicate calls to getDbType()

* chore(schema-compiler): improvement in extractFilterMembers

* chore(schema-compiler): Improvement in evaluateSymbolSql()

* fix(schema-compiler): For views, propagate FILTER_PARAMS to inner cube SELECT

* chore(schema-compiler): add tests for FILTER_PARAMS propagation

* chore(schema-compiler): fix yaml-compiler tests

* fix(schema-compiler): fix backalias resolution in FILTER_PARAMS to exclude segments

Author: KSDaemon

packages/cubejs-schema-compiler/src/adapter/BaseGroupFilter.ts

@@ -1,5 +1,3 @@
-import R from 'ramda';
-
 export class BaseGroupFilter {
   protected readonly values: any;
 
@@ -31,7 +29,7 @@ export class BaseGroupFilter {
         return null;
       }
       return `(${sql})`;
-    }).filter(R.identity).join(` ${this.operator.toUpperCase()} `);
+    }).filter(x => x).join(` ${this.operator.toUpperCase()} `);
 
     if (!r.length) {
       return null;

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -2180,15 +2180,16 @@ export class BaseQuery {
     const memberPathArray = [cubeName, name];
     const memberPath = this.cubeEvaluator.pathFromArray(memberPathArray);
     let type = memberExpressionType;
-    if (!type && this.cubeEvaluator.isMeasure(memberPathArray)) {
-      type = 'measure';
-    }
-    if (!type && this.cubeEvaluator.isDimension(memberPathArray)) {
-      type = 'dimension';
-    }
-    if (!type && this.cubeEvaluator.isSegment(memberPathArray)) {
-      type = 'segment';
+    if (!type) {
+      if (this.cubeEvaluator.isMeasure(memberPathArray)) {
+        type = 'measure';
+      } else if (this.cubeEvaluator.isDimension(memberPathArray)) {
+        type = 'dimension';
+      } else if (this.cubeEvaluator.isSegment(memberPathArray)) {
+        type = 'segment';
+      }
     }
+
     const parentMember = this.safeEvaluateSymbolContext().currentMember;
     if (this.safeEvaluateSymbolContext().memberChildren && parentMember) {
       this.safeEvaluateSymbolContext().memberChildren[parentMember] = this.safeEvaluateSymbolContext().memberChildren[parentMember] || [];
@@ -2358,7 +2359,7 @@ export class BaseQuery {
 
   /**
    * Evaluate escaped SQL-alias for cube or cube's property
-   * (measure, dimention).
+   * (measure, dimension).
    * @param {string} cubeName
    * @returns string
    */
@@ -3529,25 +3530,29 @@ export class BaseQuery {
   static extractFilterMembers(filter) {
     if (filter.operator === 'and' || filter.operator === 'or') {
       return filter.values.map(f => BaseQuery.extractFilterMembers(f)).reduce((a, b) => ((a && b) ? { ...a, ...b } : null), {});
-    } else if (filter.measure || filter.dimension) {
+    } else if (filter.measure) {
+      return {
+        [filter.measure]: true
+      };
+    } else if (filter.dimension) {
       return {
-        [filter.measure || filter.dimension]: true
+        [filter.dimension]: true
       };
     } else {
       return null;
     }
   }
 
-  static findAndSubTreeForFilterGroup(filter, groupMembers, newGroupFilter) {
+  static findAndSubTreeForFilterGroup(filter, groupMembers, newGroupFilter, aliases) {
     if ((filter.operator === 'and' || filter.operator === 'or') && !filter.values?.length) {
       return null;
     }
     const filterMembers = BaseQuery.extractFilterMembers(filter);
-    if (filterMembers && Object.keys(filterMembers).every(m => groupMembers.indexOf(m) !== -1)) {
+    if (filterMembers && Object.keys(filterMembers).every(m => (groupMembers.indexOf(m) !== -1 || aliases.indexOf(m) !== -1))) {
       return filter;
     }
     if (filter.operator === 'and') {
-      const result = filter.values.map(f => BaseQuery.findAndSubTreeForFilterGroup(f, groupMembers, newGroupFilter)).filter(f => !!f);
+      const result = filter.values.map(f => BaseQuery.findAndSubTreeForFilterGroup(f, groupMembers, newGroupFilter, aliases)).filter(f => !!f);
       if (!result.length) {
         return null;
       }
@@ -3572,21 +3577,30 @@ export class BaseQuery {
     );
   }
 
-  static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter) {
+  static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases) {
     if (!filter) {
       return '1 = 1';
     }
 
     if (filter.operator === 'and' || filter.operator === 'or') {
       const values = filter.values
-        .map(f => BaseQuery.renderFilterParams(f, filterParamArgs, allocateParam, newGroupFilter))
+        .map(f => BaseQuery.renderFilterParams(f, filterParamArgs, allocateParam, newGroupFilter, aliases))
         .map(v => ({ filterToWhere: () => v }));
 
       return newGroupFilter({ operator: filter.operator, values }).filterToWhere();
     }
 
-    const filterParams = filter && filter.filterParams();
-    const filterParamArg = filterParamArgs.filter(p => p.__member() === filter.measure || p.__member() === filter.dimension)[0];
+    const filterParams = filter.filterParams();
+    const filterParamArg = filterParamArgs.filter(p => {
+      const member = p.__member();
+      return member === filter.measure ||
+             member === filter.dimension ||
+            (aliases[member] && (
+              aliases[member] === filter.measure ||
+              aliases[member] === filter.dimension
+            ));
+    })[0];
+
     if (!filterParamArg) {
       throw new Error(`FILTER_PARAMS arg not found for ${filter.measure || filter.dimension}`);
     }
@@ -3619,15 +3633,25 @@ export class BaseQuery {
         return f.__member();
       });
 
-      const filter = BaseQuery.findAndSubTreeForFilterGroup(newGroupFilter({ operator: 'and', values: allFilters }), groupMembers, newGroupFilter);
+      const aliases = allFilters ?
+        allFilters
+          .map(v => (v.query ? v.query.allBackAliasMembersExceptSegments() : {}))
+          .reduce((a, b) => ({ ...a, ...b }), {})
+        : {};
+      const filter = BaseQuery.findAndSubTreeForFilterGroup(
+        newGroupFilter({ operator: 'and', values: allFilters }),
+        groupMembers,
+        newGroupFilter,
+        Object.values(aliases)
+      );
 
-      return `(${BaseQuery.renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter)})`;
+      return `(${BaseQuery.renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases)})`;
     };
   }
 
   static filterProxyFromAllFilters(allFilters, cubeEvaluator, allocateParam, newGroupFilter) {
     return new Proxy({}, {
-      get: (target, name) => {
+      get: (_target, name) => {
         if (name === '_objectWithResolvedProperties') {
           return true;
         }
@@ -3644,17 +3668,75 @@ export class BaseQuery {
                 return cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName]);
               },
               toString() {
+                // Segments should be excluded because they are evaluated separately in cubeReferenceProxy
+                // In other case this falls into the recursive loop/stack exceeded caused by:
+                // collectFrom() -> traverseSymbol() -> evaluateSymbolSql() ->
+                // evaluateSql() -> resolveSymbolsCall() -> cubeReferenceProxy->toString() ->
+                // evaluateSymbolSql() -> evaluateSql()... -> and got here again
+                const aliases = allFilters ?
+                  allFilters
+                    .map(v => (v.query ? v.query.allBackAliasMembersExceptSegments() : {}))
+                    .reduce((a, b) => ({ ...a, ...b }), {})
+                  : {};
+                // Filtering aliases that somehow relate to this cube
+                const filteredAliases = Object.entries(aliases)
+                  .filter(([key, value]) => key.startsWith(cubeNameObj.cube) || value.startsWith(cubeNameObj.cube))
+                  .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {});
                 const filter = BaseQuery.findAndSubTreeForFilterGroup(
                   newGroupFilter({ operator: 'and', values: allFilters }),
                   [cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName])],
-                  newGroupFilter
+                  newGroupFilter,
+                  Object.values(filteredAliases)
                 );
-                return `(${BaseQuery.renderFilterParams(filter, [this], allocateParam, newGroupFilter)})`;
+
+                return `(${BaseQuery.renderFilterParams(filter, [this], allocateParam, newGroupFilter, aliases)})`;
               }
             })
           })
         });
       }
     });
   }
+
+  flattenAllMembers(excludeSegments = false) {
+    return R.flatten(
+      this.measures
+        .concat(this.dimensions)
+        .concat(excludeSegments ? [] : this.segments)
+        .concat(this.filters)
+        .concat(this.measureFilters)
+        .concat(this.timeDimensions)
+        .map(m => m.getMembers()),
+    );
+  }
+
+  allBackAliasMembersExceptSegments() {
+    return this.backAliasMembers(this.flattenAllMembers(true));
+  }
+
+  allBackAliasMembers() {
+    return this.backAliasMembers(this.flattenAllMembers());
+  }
+
+  backAliasMembers(members) {
+    const query = this;
+    return members.map(
+      member => {
+        const collectedMembers = query
+          .collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor');
+        const memberPath = member.expressionPath();
+        let nonAliasSeen = false;
+        return collectedMembers
+          .filter(d => {
+            if (!query.cubeEvaluator.byPathAnyType(d).aliasMember) {
+              nonAliasSeen = true;
+            }
+            return !nonAliasSeen;
+          })
+          .map(d => (
+            { [query.cubeEvaluator.byPathAnyType(d).aliasMember]: memberPath }
+          )).reduce((a, b) => ({ ...a, ...b }), {});
+      }
+    ).reduce((a, b) => ({ ...a, ...b }), {});
+  }
 }

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -157,7 +157,7 @@ export class PreAggregations {
     const queryForSqlEvaluation = this.query.preAggregationQueryForSqlEvaluation(cube, preAggregation);
     const partitionInvalidateKeyQueries = queryForSqlEvaluation.partitionInvalidateKeyQueries && queryForSqlEvaluation.partitionInvalidateKeyQueries(cube, preAggregation);
 
-    const allBackAliasMembers = PreAggregations.allBackAliasMembers(this.query);
+    const allBackAliasMembers = this.query.allBackAliasMembers();
 
     const matchedTimeDimension = preAggregation.partitionGranularity && !this.hasCumulativeMeasures &&
       this.query.timeDimensions.find(td => {
@@ -292,7 +292,7 @@ export class PreAggregations {
   static transformQueryToCanUseForm(query) {
     const flattenDimensionMembers = this.flattenDimensionMembers(query);
     const sortedDimensions = this.squashDimensions(query, flattenDimensionMembers);
-    const allBackAliasMembers = this.allBackAliasMembers(query);
+    const allBackAliasMembers = query.allBackAliasMembers();
     const measures = query.measures.concat(query.measureFilters);
     const measurePaths = R.uniq(this.flattenMembers(measures).map(m => m.expressionPath()));
     const collectLeafMeasures = query.collectLeafMeasures.bind(query);
@@ -426,31 +426,6 @@ export class PreAggregations {
     );
   }
 
-  static backAliasMembers(query, members) {
-    return members.map(
-      member => {
-        const collectedMembers = query
-          .collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor');
-        const memberPath = member.expressionPath();
-        let nonAliasSeen = false;
-        return collectedMembers
-          .filter(d => {
-            if (!query.cubeEvaluator.byPathAnyType(d).aliasMember) {
-              nonAliasSeen = true;
-            }
-            return !nonAliasSeen;
-          })
-          .map(d => (
-            { [query.cubeEvaluator.byPathAnyType(d).aliasMember]: memberPath }
-          )).reduce((a, b) => ({ ...a, ...b }), {});
-      }
-    ).reduce((a, b) => ({ ...a, ...b }), {});
-  }
-
-  static allBackAliasMembers(query) {
-    return this.backAliasMembers(query, this.flattenAllMembers(query));
-  }
-
   static sortTimeDimensionsWithRollupGranularity(timeDimensions) {
     return timeDimensions && R.sortBy(
       R.prop(0),
@@ -750,18 +725,6 @@ export class PreAggregations {
     );
   }
 
-  static flattenAllMembers(query) {
-    return R.flatten(
-      query.measures
-        .concat(query.dimensions)
-        .concat(query.segments)
-        .concat(query.filters)
-        .concat(query.measureFilters)
-        .concat(query.timeDimensions)
-        .map(m => m.getMembers()),
-    );
-  }
-
   // eslint-disable-next-line no-unused-vars
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getCubeLattice(cube, preAggregationName, preAggregation) {