WIP: new style access policy framework for Cube

Author: bsod90

.gitignore

@@ -23,4 +23,4 @@ testings/
 rust/cubesql/profile.json
 .cubestore
 .env
-
+.vimspector.json

packages/cubejs-api-gateway/src/gateway.ts

@@ -234,7 +234,7 @@ class ApiGateway {
       const { query, variables } = req.body;
       const compilerApi = await this.getCompilerApi(req.context);
 
-      const metaConfig = await compilerApi.metaConfig({
+      const metaConfig = await compilerApi.metaConfig(req.context, {
         requestId: req.context.requestId,
       });
 
@@ -267,7 +267,7 @@ class ApiGateway {
         const compilerApi = await this.getCompilerApi(req.context);
         let schema = compilerApi.getGraphQLSchema();
         if (!schema) {
-          let metaConfig = await compilerApi.metaConfig({
+          let metaConfig = await compilerApi.metaConfig(req.context, {
             requestId: req.context.requestId,
           });
           metaConfig = this.filterVisibleItemsInMeta(req.context, metaConfig);
@@ -551,7 +551,7 @@ class ApiGateway {
     try {
       await this.assertApiScope('meta', context.securityContext);
       const compilerApi = await this.getCompilerApi(context);
-      const metaConfig = await compilerApi.metaConfig({
+      const metaConfig = await compilerApi.metaConfig(context, {
         requestId: context.requestId,
         includeCompilerId: includeCompilerId || onlyCompilerId
       });
@@ -587,7 +587,7 @@ class ApiGateway {
     try {
       await this.assertApiScope('meta', context.securityContext);
       const compilerApi = await this.getCompilerApi(context);
-      const metaConfigExtended = await compilerApi.metaConfigExtended({
+      const metaConfigExtended = await compilerApi.metaConfigExtended(context, {
         requestId: context.requestId,
       });
       const { metaConfig, cubeDefinitions } = metaConfigExtended;
@@ -1010,7 +1010,7 @@ class ApiGateway {
           } else {
             const metaCacheKey = JSON.stringify(ctx);
             if (!metaCache.has(metaCacheKey)) {
-              metaCache.set(metaCacheKey, await compiler.metaConfigExtended(ctx));
+              metaCache.set(metaCacheKey, await compiler.metaConfigExtended(context, ctx));
             }
 
             // checking and fetching result status
@@ -1180,6 +1180,7 @@ class ApiGateway {
     }, context);
 
     const startTime = new Date().getTime();
+    const compilerApi = await this.getCompilerApi(context);
 
     let normalizedQueries: NormalizedQuery[] = await Promise.all(
       queries.map(
@@ -1195,8 +1196,14 @@ class ApiGateway {
           }
 
           const normalizedQuery = normalizeQuery(currentQuery, persistent);
-          let rewrittenQuery = await this.queryRewrite(
+          // First apply cube/view level security policies
+          const queryWithRlsFilters = await compilerApi.applyRowLevelSecurity(
             normalizedQuery,
+            context
+          );
+          // Then apply user-supplied queryRewrite
+          let rewrittenQuery = await this.queryRewrite(
+            queryWithRlsFilters,
             context,
           );
 
@@ -1693,7 +1700,7 @@ class ApiGateway {
         await this.getNormalizedQueries(query, context);
 
       let metaConfigResult = await (await this
-        .getCompilerApi(context)).metaConfig({
+        .getCompilerApi(context)).metaConfig(request.context, {
         requestId: context.requestId
       });
 
@@ -1803,7 +1810,7 @@ class ApiGateway {
         await this.getNormalizedQueries(query, context, request.streaming, request.memberExpressions);
 
       const compilerApi = await this.getCompilerApi(context);
-      let metaConfigResult = await compilerApi.metaConfig({
+      let metaConfigResult = await compilerApi.metaConfig(request.context, {
         requestId: context.requestId
       });
 

packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts

@@ -48,7 +48,8 @@ const annotation = (
 ) => (member: string | MemberExpression): undefined | [string, ConfigItem] => {
   const [cubeName, fieldName] = (<MemberExpression>member).expression ? [(<MemberExpression>member).cubeName, (<MemberExpression>member).name] : (<string>member).split('.');
   const memberWithoutGranularity = [cubeName, fieldName].join('.');
-  const config: ConfigItem = configMap[cubeName][memberType]
+  const cubeConfig = configMap[cubeName];
+  const config: ConfigItem = cubeConfig && cubeConfig[memberType]
     .find(m => m.name === memberWithoutGranularity);
 
   if (!config) {

packages/cubejs-api-gateway/test/index.test.ts

@@ -471,7 +471,7 @@ describe('API Gateway', () => {
         queryRewrite: async (query, _context) => {
           query.limit = 2;
           return query;
-        }
+        },
       }
     );
 

packages/cubejs-api-gateway/test/mocks.ts

@@ -75,6 +75,10 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
     return 'postgres';
   },
 
+  async applyRowLevelSecurity(query: any) {
+    return query;
+  },
+
   async metaConfig() {
     return [
       {

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -112,9 +112,71 @@ export class CubeEvaluator extends CubeSymbols {
 
     this.prepareHierarchies(cube);
 
+    this.prepareAccessPolicy(cube, errorReporter);
+
     return cube;
   }
 
+  private allMembersOrList(cube: any, specifier: string | string[]): string[] {
+    const types = ['measures', 'dimensions', 'segments'];
+    if (specifier === '*') {
+      const allMembers = R.unnest(types.map(type => Object.keys(cube[type] || {})));
+      return allMembers;
+    } else {
+      return specifier as string[] || [];
+    }
+  }
+
+  private prepareAccessPolicy(cube: any, errorReporter: ErrorReporter) {
+    if (!cube.accessPolicy) {
+      return;
+    }
+
+    const memberMapper = (memberType: string) => (member: string) => {
+      if (member.indexOf('.') !== -1) {
+        const cubeName = member.split('.')[0];
+        if (cubeName !== cube.name) {
+          errorReporter.error(
+            `Paths aren't allowed in the accessPolicy policy but '${member}' provided as ${memberType} for ${cube.name}`
+          );
+        }
+        return member;
+      }
+      return this.pathFromArray([cube.name, member]);
+    };
+
+    const filterEvaluator = (filter: any) => {
+      if (filter.member) {
+        filter.memberReference = this.evaluateReferences(cube.name, filter.member);
+        filter.memberReference = memberMapper('a filter member reference')(filter.memberReference);
+      } else {
+        if (filter.and) {
+          filter.and.forEach(filterEvaluator);
+        }
+        if (filter.or) {
+          filter.or.forEach(filterEvaluator);
+        }
+      }
+    };
+
+    for (const policy of cube.accessPolicy) {
+      for (const filter of policy?.rowLevel?.filters || []) {
+        filterEvaluator(filter);
+      }
+
+      if (policy.memberLevel) {
+        policy.memberLevel.includesMembers = this.allMembersOrList(
+          cube,
+          policy.memberLevel.includes
+        ).map(memberMapper('an includes member'));
+        policy.memberLevel.excludesMembers = this.allMembersOrList(
+          cube,
+          policy.memberLevel.excludes || []
+        ).map(memberMapper('an excludes member'));
+      }
+    }
+  }
+
   private prepareHierarchies(cube: any) {
     if (Array.isArray(cube.hierarchies)) {
       cube.hierarchies = cube.hierarchies.map(hierarchy => ({

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -10,6 +10,9 @@ import { BaseQuery } from '../adapter';
 const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/;
 const CONTEXT_SYMBOLS = {
   SECURITY_CONTEXT: 'securityContext',
+  // SECURITY_CONTEXT has been deprecated, however security_context (lowecase)
+  // is allowed in RBAC policies for query-time attribute matching
+  security_context: 'securityContext',
   FILTER_PARAMS: 'filterParams',
   FILTER_GROUP: 'filterGroup',
   SQL_UTILS: 'sqlUtils'
@@ -139,6 +142,7 @@ export class CubeSymbols {
     this.camelCaseTypes(cube.dimensions);
     this.camelCaseTypes(cube.segments);
     this.camelCaseTypes(cube.preAggregations);
+    this.camelCaseTypes(cube.accessPolicy);
 
     if (cube.preAggregations) {
       this.transformPreAggregations(cube.preAggregations);
@@ -406,6 +410,34 @@ export class CubeSymbols {
     });
   }
 
+  /**
+   * This method is mainly used for evaluating RLS conditions and filters.
+   * It allows referencing security_context (lowecase) in dynamic conditions or filter values.
+   *
+   * It currently does not support async calls because inner resolveSymbol and
+   * resolveSymbolsCall are sync. Async support may be added later with deeper
+   * refactoring.
+   */
+  evaluateContextFunction(cube, contextFn, context = {}) {
+    const cubeEvaluator = this;
+
+    const res = cubeEvaluator.resolveSymbolsCall(contextFn, (name) => {
+      const resolvedSymbol = this.resolveSymbol(cube, name);
+      if (resolvedSymbol) {
+        return resolvedSymbol;
+      }
+      throw new UserError(
+        `Cube references are not allowed when evaluating RLS conditions or filters. Found: ${name} in ${cube.name}`
+      );
+    }, {
+      contextSymbols: {
+        securityContext: context.securityContext,
+      }
+    });
+
+    return res;
+  }
+
   evaluateReferences(cube, referencesFn, options = {}) {
     const cubeEvaluator = this;
 

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -25,7 +25,7 @@ export const nonStringFields = new Set([
   'external',
   'useOriginalSqlPreAggregations',
   'readOnly',
-  'prefix'
+  'prefix',
 ]);
 
 const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/;
@@ -615,6 +615,63 @@ const SegmentsSchema = Joi.object().pattern(identifierRegex, Joi.object().keys({
   public: Joi.boolean().strict(),
 }));
 
+const PolicyFilterSchema = Joi.object().keys({
+  member: Joi.func().required(),
+  memberReference: Joi.string(),
+  operator: Joi.any().valid(
+    'equals',
+    'notEquals',
+    'contains',
+    'notContains',
+    'startsWith',
+    'notStartsWith',
+    'endsWith',
+    'notEndsWith',
+    'gt',
+    'gte',
+    'lt',
+    'lte',
+    'inDateRange',
+    'notInDateRange',
+    'beforeDate',
+    'beforeOrOnDate',
+    'afterDate',
+    'afterOrOnDate',
+  ).required(),
+  values: Joi.func().required(),
+});
+
+const PolicyFilterConditionSchema = Joi.object().keys({
+  or: Joi.array().items(PolicyFilterSchema, Joi.link('...').description('Filter Condition schema')),
+  and: Joi.array().items(PolicyFilterSchema, Joi.link('...').description('Filter Condition schema')),
+}).xor('or', 'and');
+
+const MemberLevelPolicySchema = Joi.object().keys({
+  includes: Joi.alternatives([
+    Joi.string().valid('*'),
+    Joi.array().items(Joi.string())
+  ]).required(),
+  excludes: Joi.alternatives([
+    Joi.array().items(Joi.string().required())
+  ]),
+  includesMembers: Joi.array().items(Joi.string().required()),
+  excludesMembers: Joi.array().items(Joi.string().required()),
+});
+
+const RowLevelPolicySchema = Joi.object().keys({
+  filters: Joi.array().items(PolicyFilterSchema, PolicyFilterConditionSchema),
+  allowAll: Joi.boolean().valid(true).strict(),
+}).xor('filters', 'allowAll');
+
+const RolePolicySchema = Joi.object().keys({
+  role: Joi.string().required(),
+  memberLevel: MemberLevelPolicySchema,
+  rowLevel: RowLevelPolicySchema.required(),
+  conditions: Joi.array().items(Joi.object().keys({
+    if: Joi.func().required(),
+  })),
+});
+
 /* *****************************
  * ATTENTION:
  * In case of adding/removing/changing any Joi.func() field that needs to be transpiled,
@@ -692,6 +749,7 @@ const baseSchema = {
     title: Joi.string(),
     levels: Joi.func()
   })),
+  accessPolicy: Joi.array().items(RolePolicySchema.required()),
 };
 
 const cubeSchema = inherit(baseSchema, {
@@ -726,6 +784,7 @@ const viewSchema = inherit(baseSchema, {
       'object.oxor': 'Using split together with prefix is not supported'
     })
   ),
+  accessPolicy: Joi.array().items(RolePolicySchema.required()),
 });
 
 function formatErrorMessageFromDetails(explain, d) {

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -146,7 +146,20 @@ export class YamlCompiler {
             return this.parsePythonIntoArrowFunction(obj, cubeName, obj, errorsReport);
           } else if (Array.isArray(obj)) {
             const resultAst = t.program([t.expressionStatement(t.arrayExpression(obj.map(code => {
-              const ast = this.parsePythonAndTranspileToJs(code, errorsReport);
+              let ast: t.Program | t.NullLiteral | t.BooleanLiteral | t.NumericLiteral | null = null;
+              // Special case for accessPolicy.rowLevel.filter.values and other values-like fields
+              if (propertyPath[propertyPath.length - 1] === 'values') {
+                if (typeof code === 'string') {
+                  ast = this.parsePythonAndTranspileToJs(`f"${this.escapeDoubleQuotes(code)}"`, errorsReport);
+                } else if (typeof code === 'boolean') {
+                  ast = t.booleanLiteral(code);
+                } else if (typeof code === 'number') {
+                  ast = t.numericLiteral(code);
+                }
+              }
+              if (ast === null) {
+                ast = this.parsePythonAndTranspileToJs(code, errorsReport);
+              }
               return this.extractProgramBodyIfNeeded(ast);
             }).filter(ast => !!ast)))]);
             return this.astIntoArrowFunction(resultAst, '', cubeName);

packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts

@@ -25,6 +25,9 @@ export const transpiledFieldsPatterns: Array<RegExp> = [
   /^excludes$/,
   /^hierarchies\.[0-9]+\.levels$/,
   /^cubes\.[0-9]+\.(joinPath|join_path)$/,
+  /^(accessPolicy|access_policy)\.[0-9]+\.(rowLevel|row_level)\.filters\.[0-9]+.*\.member$/,
+  /^(accessPolicy|access_policy)\.[0-9]+\.(rowLevel|row_level)\.filters\.[0-9]+.*\.values$/,
+  /^(accessPolicy|access_policy)\.[0-9]+\.conditions.[0-9]+\.if$/,
 ];
 
 export const transpiledFields: Set<String> = new Set<String>();