feat(core,build): add sanitizer and parameterizer

- passing context to filter runner
- add santizier filter builder and runner
- add TemplateInput class to control parameterize logics
- update req runner to inject parameterizers

Author: oscar60310

labs/playground1/sqls/artist/artist.yaml

@@ -5,3 +5,5 @@ request:
     description: constituent id
     validators:
       - required
+exampleParameter:
+  id: "1"

labs/playground1/sqls/artist/works.yaml

@@ -5,3 +5,5 @@ request:
     description: constituent id
     validators:
       - required
+exampleParameter:
+  id: '1'

packages/build/src/lib/schema-parser/middleware/responseSampler.ts

@@ -2,6 +2,7 @@ import { inject } from 'inversify';
 import { RawAPISchema, SchemaParserMiddleware } from './middleware';
 import {
   APISchema,
+  DataSource,
   FieldDataType,
   ResponseProperty,
   TemplateEngine,
@@ -11,12 +12,15 @@ import { unionBy } from 'lodash';
 
 export class ResponseSampler extends SchemaParserMiddleware {
   private templateEngine: TemplateEngine;
+  private dataSource: DataSource;
 
   constructor(
-    @inject(CORE_TYPES.TemplateEngine) templateEngine: TemplateEngine
+    @inject(CORE_TYPES.TemplateEngine) templateEngine: TemplateEngine,
+    @inject(CORE_TYPES.DataSource) dataSource: DataSource
   ) {
     super();
     this.templateEngine = templateEngine;
+    this.dataSource = dataSource;
   }
 
   public async handle(
@@ -27,17 +31,19 @@ export class ResponseSampler extends SchemaParserMiddleware {
     const schema = rawSchema as APISchema;
     if (!schema.exampleParameter) return;
 
+    const prepared = await this.dataSource.prepare(schema.exampleParameter);
+
     const response = await this.templateEngine.execute(
       schema.templateSource,
-      { context: { params: schema.exampleParameter } },
+      { ['_prepared']: prepared },
       // We only need the columns of this query, so we set offset/limit both to 0 here.
       {
         limit: 0,
         offset: 0,
       }
     );
     // TODO: I haven't known the response of queryBuilder.value(), assume that there is a "columns" property that indicates the columns' name and type here.
-    const columns: { name: string; type: string }[] = response.columns;
+    const columns: { name: string; type: string }[] = response.getColumns();
     const responseColumns = this.normalizeResponseColumns(columns);
     schema.response = this.mergeResponse(
       schema.response || [],

packages/core/src/lib/data-query/executor.ts

@@ -8,6 +8,8 @@ export interface IExecutor {
     query: string,
     bindParams: BindParameters
   ): Promise<IDataQueryBuilder>;
+
+  getDataSource(): Promise<DataSource>;
 }
 
 @injectable()
@@ -16,6 +18,11 @@ export class QueryExecutor implements IExecutor {
   constructor(@inject(TYPES.DataSource) dataSource: DataSource) {
     this.dataSource = dataSource;
   }
+
+  public async getDataSource() {
+    return this.dataSource;
+  }
+
   /**
    * create data query builder
    * @returns

packages/core/src/lib/data-source/pg.ts

@@ -5,7 +5,7 @@ import {
   DataSource,
   ExecuteOptions,
   IdentifierParameters,
-  RequestParameters,
+  RequestParameter,
   VulcanExtensionId,
   VulcanInternalExtension,
 } from '../../models/extensions';
@@ -24,19 +24,8 @@ export class PGDataSource extends DataSource {
       },
     };
   }
-  public async prepare(params: RequestParameters) {
-    const identifiers = {} as IdentifierParameters;
-    const binds = {} as BindParameters;
-    let index = 1;
-    for (const key of Object.keys(params)) {
-      const identifier = `$${index}`;
-      identifiers[key] = identifier;
-      binds[identifier] = params[key];
-      index += 1;
-    }
-    return {
-      identifiers,
-      binds,
-    };
+
+  public async prepare({ parameterIndex }: RequestParameter) {
+    return `$${parameterIndex}`;
   }
 }

packages/core/src/lib/template-engine/built-in-extensions/index.ts

@@ -4,3 +4,5 @@ import SqlHelper from './sql-helper';
 import Validator from './validator';
 
 export default [CustomError, QueryBuilder, SqlHelper, Validator];
+
+export * from './query-builder';

packages/core/src/lib/template-engine/built-in-extensions/query-builder/constants.ts

@@ -3,3 +3,6 @@ export const METADATA_NAME = 'builder.vulcan.com';
 export const EXECUTE_COMMAND_NAME = 'value';
 export const EXECUTE_FILTER_NAME = 'execute';
 export const REFERENCE_SEARCH_MAX_DEPTH = 100;
+export const SANITIZE_SOURCES = ['context'];
+export const SANITIZER_NAME = 'sanitize';
+export const PARAMETERIZER_VAR_NAME = 'parameterizer';

packages/core/src/lib/template-engine/built-in-extensions/query-builder/executorRunner.ts

@@ -1,12 +1,18 @@
 import { IDataQueryBuilder } from '@vulcan-sql/core/data-query';
-import { FilterRunner, VulcanInternalExtension } from '@vulcan-sql/core/models';
+import {
+  FilterRunner,
+  FilterRunnerTransformOptions,
+  VulcanInternalExtension,
+} from '@vulcan-sql/core/models';
 import { EXECUTE_FILTER_NAME } from './constants';
 
 @VulcanInternalExtension()
 export class ExecutorRunner extends FilterRunner {
   public filterName = EXECUTE_FILTER_NAME;
 
-  public async transform({ value }: { value: any; args: any[] }): Promise<any> {
+  public async transform({
+    value,
+  }: FilterRunnerTransformOptions): Promise<any> {
     const builder: IDataQueryBuilder = value;
     return builder.value();
   }

packages/core/src/lib/template-engine/built-in-extensions/query-builder/index.ts

@@ -2,5 +2,16 @@ import { ReqTagBuilder } from './reqTagBuilder';
 import { ReqTagRunner } from './reqTagRunner';
 import { ExecutorRunner } from './executorRunner';
 import { ExecutorBuilder } from './executorBuilder';
+import { SanitizerBuilder } from './sanitizerBuilder';
+import { SanitizerRunner } from './sanitizerRunner';
 
-export default [ReqTagBuilder, ReqTagRunner, ExecutorRunner, ExecutorBuilder];
+export default [
+  ReqTagBuilder,
+  ReqTagRunner,
+  ExecutorRunner,
+  ExecutorBuilder,
+  SanitizerBuilder,
+  SanitizerRunner,
+];
+
+export * from './templateInput';

packages/core/src/lib/template-engine/built-in-extensions/query-builder/parameterizer.ts

@@ -0,0 +1,36 @@
+import { DataSource } from '@vulcan-sql/core/models';
+
+export class Parameterizer {
+  private parameterIndex = 1;
+  private sealed = false;
+  // We MUST not use pure object here because we care about the order of the keys.
+  // https://stackoverflow.com/questions/5525795/does-javascript-guarantee-object-property-order
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map#description
+  private idToValueMapping = new Map<string, any>();
+  private dataSource: DataSource;
+
+  constructor(dataSource: DataSource) {
+    this.dataSource = dataSource;
+  }
+
+  public async generateIdentifier(value: any): Promise<string> {
+    if (this.sealed)
+      throw new Error(
+        `This parameterizer has been sealed, we might use the parameterizer from a wrong request scope.`
+      );
+    const id = await this.dataSource.prepare({
+      parameterIndex: this.parameterIndex++,
+      value,
+    });
+    this.idToValueMapping.set(id, value);
+    return id;
+  }
+
+  public seal() {
+    this.sealed = true;
+  }
+
+  public getBinding() {
+    return this.idToValueMapping;
+  }
+}

packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagRunner.ts

@@ -6,7 +6,8 @@ import {
   TagRunnerOptions,
   VulcanInternalExtension,
 } from '@vulcan-sql/core/models';
-import { FINIAL_BUILDER_NAME } from './constants';
+import { FINIAL_BUILDER_NAME, PARAMETERIZER_VAR_NAME } from './constants';
+import { Parameterizer } from './parameterizer';
 
 @VulcanInternalExtension()
 export class ReqTagRunner extends TagRunner {
@@ -24,16 +25,26 @@ export class ReqTagRunner extends TagRunner {
 
   public async run({ context, args, contentArgs }: TagRunnerOptions) {
     const name = args[0];
+
+    const dataSource = await this.executor.getDataSource();
+
+    const parameterizer = new Parameterizer(dataSource);
+    // parameterizer from parent, we should set it back after rendered our context.
+    const parentParameterizer = context.lookup(PARAMETERIZER_VAR_NAME);
+    context.setVariable(PARAMETERIZER_VAR_NAME, parameterizer);
     let query = '';
     for (let index = 0; index < contentArgs.length; index++) {
       query += await contentArgs[index]();
     }
+    // Seal current parameterizer to avoid incorrect usage.
+    parameterizer.seal();
+    context.setVariable(PARAMETERIZER_VAR_NAME, parentParameterizer);
     query = query
       .split(/\r?\n/)
       .filter((line) => line.trim().length > 0)
       .join('\n');
     // Get bind real parameters and pass to data query builder for data source used.
-    const binds = (context.ctx || {})['_paramBinds'] || {};
+    const binds = parameterizer.getBinding();
     const builder = await this.executor.createBuilder(query, binds);
     context.setVariable(name, builder);
 

packages/core/src/lib/template-engine/built-in-extensions/query-builder/sanitizerBuilder.ts

@@ -0,0 +1,78 @@
+import {
+  FilterBuilder,
+  VulcanInternalExtension,
+} from '@vulcan-sql/core/models';
+import * as nunjucks from 'nunjucks';
+import { visitChildren } from '../../extension-utils';
+import {
+  REFERENCE_SEARCH_MAX_DEPTH,
+  SANITIZER_NAME,
+  SANITIZE_SOURCES,
+} from './constants';
+
+@VulcanInternalExtension()
+export class SanitizerBuilder extends FilterBuilder {
+  public filterName = SANITIZER_NAME;
+  public override onVisit(node: nunjucks.nodes.Node): void {
+    if (node instanceof nunjucks.nodes.Root) this.addSanitizer(node);
+  }
+
+  private addSanitizer(node: nunjucks.nodes.Node, parentHasOutputNode = false) {
+    visitChildren(node, (child, replace) => {
+      if (child instanceof nunjucks.nodes.LookupVal) {
+        const source = this.findSourceOfLookUpNode(child);
+        if (SANITIZE_SOURCES.includes(source.value)) {
+          const filter = new nunjucks.nodes.Filter(node.lineno, node.colno);
+          filter.name = new nunjucks.nodes.Symbol(
+            node.lineno,
+            node.colno,
+            SANITIZER_NAME
+          );
+          const args = new nunjucks.nodes.NodeList(node.lineno, node.colno);
+          // The first argument is the target of the filter
+          args.addChild(child);
+          // The second argument indicates whether it should be parameterized, we only parameterize parameters when once of parent nodes is a Output node.
+          const shouldBeParameterized = new nunjucks.nodes.Literal(
+            node.lineno,
+            node.colno,
+            `${parentHasOutputNode || node instanceof nunjucks.nodes.Output}`
+          );
+          args.addChild(shouldBeParameterized);
+          filter.args = args;
+          replace(filter);
+        }
+      } else {
+        this.addSanitizer(
+          child,
+          parentHasOutputNode || node instanceof nunjucks.nodes.Output
+        );
+      }
+    });
+  }
+
+  private findSourceOfLookUpNode(
+    node: nunjucks.nodes.LookupVal
+  ): nunjucks.nodes.Symbol {
+    let depth = 0;
+    let source: typeof node.target = node.target;
+    while (!(source instanceof nunjucks.nodes.Symbol)) {
+      depth++;
+      if (depth > REFERENCE_SEARCH_MAX_DEPTH) {
+        throw new Error('Max depth reached');
+      }
+
+      if (source instanceof nunjucks.nodes.LookupVal) {
+        // LookupVal: parent.source
+        source = source.target;
+      } else {
+        // FunCall: parent().source
+        source = source.name;
+      }
+
+      if (!source) {
+        throw new Error(`Can find the source of node ${node}`);
+      }
+    }
+    return source;
+  }
+}

packages/core/src/lib/template-engine/built-in-extensions/query-builder/sanitizerRunner.ts

@@ -0,0 +1,31 @@
+import {
+  FilterRunner,
+  FilterRunnerTransformOptions,
+  VulcanInternalExtension,
+} from '@vulcan-sql/core/models';
+import { PARAMETERIZER_VAR_NAME, SANITIZER_NAME } from './constants';
+import { TemplateInput } from './templateInput';
+
+@VulcanInternalExtension()
+export class SanitizerRunner extends FilterRunner {
+  public filterName = SANITIZER_NAME;
+
+  public async transform({
+    value,
+    args,
+    context,
+  }: FilterRunnerTransformOptions): Promise<any> {
+    let input: TemplateInput;
+    // Wrap the value to template input to parameterized
+    if (value instanceof TemplateInput) input = value;
+    else {
+      input = new TemplateInput(value);
+    }
+
+    const parameterizer = context.lookup(PARAMETERIZER_VAR_NAME);
+    if (!parameterizer) throw new Error(`No parameterizer found`);
+    return args[0] === 'true'
+      ? await input.parameterize(parameterizer)
+      : input.raw();
+  }
+}

packages/core/src/lib/template-engine/built-in-extensions/query-builder/templateInput.ts

@@ -0,0 +1,17 @@
+import { Parameterizer } from './parameterizer';
+
+export class TemplateInput {
+  private rawValue: any;
+
+  constructor(rawValue: any) {
+    this.rawValue = rawValue;
+  }
+
+  public raw() {
+    return this.rawValue;
+  }
+
+  public parameterize(parameterizer: Parameterizer) {
+    return parameterizer.generateIdentifier(this.rawValue);
+  }
+}

packages/core/src/lib/template-engine/built-in-extensions/sql-helper/uniqueFilterRunner.ts

@@ -1,4 +1,8 @@
-import { FilterRunner, VulcanInternalExtension } from '@vulcan-sql/core/models';
+import {
+  FilterRunner,
+  FilterRunnerTransformOptions,
+  VulcanInternalExtension,
+} from '@vulcan-sql/core/models';
 import { uniq, uniqBy } from 'lodash';
 
 @VulcanInternalExtension()
@@ -7,10 +11,7 @@ export class UniqueFilterRunner extends FilterRunner {
   public async transform({
     value,
     args,
-  }: {
-    value: any[];
-    args: any[];
-  }): Promise<any> {
+  }: FilterRunnerTransformOptions): Promise<any> {
     if (args.length === 0) {
       return uniq(value);
     }