feat: add support for index selectors (#13)

New syntax supported .a.0.1 as an alternative to .a[0][1]. This is mainly done as source mapping files uses this syntax.
rudderlabs · Nov 30, 2022 · db8a661 · db8a661
1 parent 4fc59c0
commit db8a661
Show file tree

Hide file tree

Showing 9 changed files with 138 additions and 78 deletions.
diff --git a/src/lexer.ts b/src/lexer.ts
@@ -43,6 +43,10 @@ export class JsonTemplateLexer {
     return JsonTemplateLexer.isLiteralToken(this.lookahead());
   }
 
+  matchINT(steps = 0): boolean {
+    return this.matchTokenType(TokenType.INT, steps);
+  }
+
   matchToArray(): boolean {
     return this.match('[') && this.match(']', 1);
   }
@@ -58,8 +62,7 @@ export class JsonTemplateLexer {
   matchPathPartSelector(): boolean {
     let token = this.lookahead();
     if (token.type === TokenType.PUNCT) {
-      let value = token.value;
-      return value === '.' || value === '..';
+      return token.value === '.' || token.value === '..';
     }
     return false;
   }
@@ -91,41 +94,37 @@ export class JsonTemplateLexer {
     return Object.values(Keyword).some((op) => op.toString() === id);
   }
 
-  matchKeyword(op: string): boolean {
+  matchKeyword(): boolean {
+    return this.matchTokenType(TokenType.KEYWORD);
+  }
+
+  matchKeywordValue(val: string): boolean {
     let token = this.lookahead();
-    return token.type === TokenType.KEYWORD && token.value === op;
+    return token.type === TokenType.KEYWORD && token.value === val;
   }
 
   matchIN(): boolean {
-    return this.matchKeyword(Keyword.IN);
+    return this.matchKeywordValue(Keyword.IN);
   }
 
   matchFunction(): boolean {
-    return this.matchKeyword(Keyword.FUNCTION);
+    return this.matchKeywordValue(Keyword.FUNCTION);
   }
 
   matchNew(): boolean {
-    return this.matchKeyword(Keyword.NEW);
+    return this.matchKeywordValue(Keyword.NEW);
   }
 
   matchTypeOf(): boolean {
-    return this.matchKeyword(Keyword.TYPEOF);
+    return this.matchKeywordValue(Keyword.TYPEOF);
   }
 
   matchAwait(): boolean {
-    return this.matchKeyword(Keyword.AWAIT);
-  }
-
-  matchAsync(): boolean {
-    return this.matchKeyword(Keyword.ASYNC);
+    return this.matchKeywordValue(Keyword.AWAIT);
   }
 
   matchLambda(): boolean {
-    return this.matchKeyword(Keyword.LAMBDA);
-  }
-
-  matchDefinition(): boolean {
-    return this.matchKeyword(Keyword.LET) || this.matchKeyword(Keyword.CONST);
+    return this.matchKeywordValue(Keyword.LAMBDA);
   }
 
   expect(value) {
@@ -217,7 +216,7 @@ export class JsonTemplateLexer {
       };
     }
 
-    let token = this.scanPunctuator() || this.scanID() || this.scanString() || this.scanNumeric();
+    let token = this.scanPunctuator() || this.scanID() || this.scanString() || this.scanInteger();
     if (token) {
       return token;
     }
@@ -250,7 +249,8 @@ export class JsonTemplateLexer {
   static isLiteralToken(token: Token) {
     return (
       token.type === TokenType.BOOL ||
-      token.type === TokenType.NUM ||
+      token.type === TokenType.INT ||
+      token.type === TokenType.FLOAT ||
       token.type === TokenType.STR ||
       token.type === TokenType.NULL ||
       token.type === TokenType.UNDEFINED
@@ -390,33 +390,25 @@ export class JsonTemplateLexer {
     this.throwUnexpectedToken();
   }
 
-  private scanNumeric(): Token | undefined {
-    let start = this.idx,
-      ch = this.codeChars[this.idx],
-      isFloat = ch === '.';
-
-    if (isFloat || JsonTemplateLexer.isDigit(ch)) {
-      let num = ch;
-      while (++this.idx < this.codeChars.length) {
-        ch = this.codeChars[this.idx];
-        if (ch === '.') {
-          if (isFloat) {
-            this.throwUnexpectedToken();
-          }
-          isFloat = true;
-        } else if (!JsonTemplateLexer.isDigit(ch)) {
-          break;
-        }
-
-        num += ch;
+  scanInteger(): Token | undefined {
+    let start = this.idx;
+    let ch = this.codeChars[this.idx];
+    if (!JsonTemplateLexer.isDigit(ch)) {
+      return;
+    }
+    let num = ch;
+    while (++this.idx < this.codeChars.length) {
+      ch = this.codeChars[this.idx];
+      if (!JsonTemplateLexer.isDigit(ch)) {
+        break;
       }
-
-      return {
-        type: TokenType.NUM,
-        value: isFloat ? parseFloat(num) : parseInt(num, 10),
-        range: [start, this.idx],
-      };
+      num += ch;
     }
+    return {
+      type: TokenType.INT,
+      value: num,
+      range: [start, this.idx],
+    };
   }
 
   private scanPunctuatorForDots(): Token | undefined {
@@ -451,9 +443,6 @@ export class JsonTemplateLexer {
         range: [start, this.idx],
       };
     } else {
-      if (JsonTemplateLexer.isDigit(ch2)) {
-        return;
-      }
       return {
         type: TokenType.PUNCT,
         value: '.',

diff --git a/src/parser.ts b/src/parser.ts
@@ -26,6 +26,7 @@ import {
   BlockExpression,
   PathOptions,
   CompileTimeExpression,
+  Keyword,
 } from './types';
 import { JsonTemplateParserError } from './errors';
 import { DATA_PARAM_KEY } from './constants';
@@ -134,7 +135,6 @@ export class JsonTemplateParser {
   private parsePathPart(): Expression | Expression[] | undefined {
     if (this.lexer.match('.()')) {
       this.lexer.ignoreTokens(1);
-      return;
     } else if (this.lexer.match('.') && this.lexer.match('(', 1)) {
       this.lexer.ignoreTokens(1);
       return this.parseBlockExpr();
@@ -251,8 +251,31 @@ export class JsonTemplateParser {
     return this.updatePathExpr(expr);
   }
 
-  private parseSelector(): SelectorExpression | Expression {
+  private createArrayIndexFilterExpr(expr: Expression): IndexFilterExpression {
+    return {
+      type: SyntaxType.ARRAY_INDEX_FILTER_EXPR,
+      indexes: {
+        type: SyntaxType.ARRAY_EXPR,
+        elements: [expr],
+      },
+    };
+  }
+
+  private createArrayFilterExpr(
+    expr: RangeFilterExpression | IndexFilterExpression,
+  ): ArrayFilterExpression {
+    return {
+      type: SyntaxType.ARRAY_FILTER_EXPR,
+      filters: [expr],
+    };
+  }
+
+  private parseSelector(): SelectorExpression | IndexFilterExpression | Expression {
     let selector = this.lexer.value();
+    if (this.lexer.matchINT()) {
+      return this.createArrayFilterExpr(this.createArrayIndexFilterExpr(this.parseLiteralExpr()));
+    }
+
     let prop: Token | undefined;
     if (this.lexer.match('*') || this.lexer.matchID() || this.lexer.matchTokenType(TokenType.STR)) {
       prop = this.lexer.lex();
@@ -629,15 +652,18 @@ export class JsonTemplateParser {
     return expr;
   }
 
-  private parseLiteralExpr(): LiteralExpression {
-    const token = this.lexer.lex();
+  private createLiteralExpr(token: Token): LiteralExpression {
     return {
       type: SyntaxType.LITERAL,
       value: token.value,
       tokenType: token.type,
     };
   }
 
+  private parseLiteralExpr(): LiteralExpression {
+    return this.createLiteralExpr(this.lexer.lex());
+  }
+
   private parseIDPath(): string {
     const idParts: string[] = [];
     while (this.lexer.matchID()) {
@@ -872,6 +898,51 @@ export class JsonTemplateParser {
     };
   }
 
+  private parseNumber(): LiteralExpression {
+    let val = this.lexer.value();
+    if (this.lexer.match('.')) {
+      val += this.lexer.value();
+      if (this.lexer.matchINT()) {
+        val += this.lexer.value();
+      }
+      return {
+        type: SyntaxType.LITERAL,
+        value: parseFloat(val),
+        tokenType: TokenType.FLOAT,
+      };
+    }
+    return {
+      type: SyntaxType.LITERAL,
+      value: parseInt(val, 10),
+      tokenType: TokenType.INT,
+    };
+  }
+
+  private parseFloatingNumber(): LiteralExpression {
+    const val = this.lexer.value() + this.lexer.value();
+    return {
+      type: SyntaxType.LITERAL,
+      value: parseFloat(val),
+      tokenType: TokenType.FLOAT,
+    };
+  }
+
+  private parseKeywordBasedExpr(): Expression {
+    const token = this.lexer.lookahead();
+    switch (token.value) {
+      case Keyword.NEW:
+        return this.parseFunctionCallExpr();
+      case Keyword.LAMBDA:
+        return this.parseLambdaExpr();
+      case Keyword.ASYNC:
+        return this.parseAsyncFunctionExpr();
+      case Keyword.FUNCTION:
+        return this.parseFunctionExpr();
+      default:
+        return this.parseDefinitionExpr();
+    }
+  }
+
   private parsePrimaryExpr(): Expression {
     if (this.lexer.match(';')) {
       return EMPTY_EXPR;
@@ -884,24 +955,16 @@ export class JsonTemplateParser {
       };
     }
 
-    if (this.lexer.matchNew()) {
-      return this.parseFunctionCallExpr();
+    if (this.lexer.matchKeyword()) {
+      return this.parseKeywordBasedExpr();
     }
 
-    if (this.lexer.matchDefinition()) {
-      return this.parseDefinitionExpr();
-    }
-
-    if (this.lexer.matchLambda()) {
-      return this.parseLambdaExpr();
-    }
-
-    if (this.lexer.matchFunction()) {
-      return this.parseFunctionExpr();
+    if (this.lexer.matchINT()) {
+      return this.parseNumber();
     }
 
-    if (this.lexer.matchAsync()) {
-      return this.parseAsyncFunctionExpr();
+    if (this.lexer.match('.') && this.lexer.matchINT(1) && !this.lexer.match('.', 2)) {
+      return this.parseFloatingNumber();
     }
 
     if (this.lexer.matchLiteral()) {

diff --git a/src/translator.ts b/src/translator.ts
@@ -681,6 +681,7 @@ export class JsonTemplateTranslator {
       } else {
         code.push(this.translateRangeFilterExpr(filter as RangeFilterExpression, dest, dest));
       }
+      code.push(`if(!${dest}){continue;}`);
     }
     return code.join('');
   }

diff --git a/src/types.ts b/src/types.ts
@@ -15,7 +15,8 @@ export enum Keyword {
 export enum TokenType {
   UNKNOWN,
   ID,
-  NUM,
+  INT,
+  FLOAT,
   STR,
   BOOL,
   NULL,

diff --git a/test/scenarios/arrays/data.ts b/test/scenarios/arrays/data.ts
@@ -2,6 +2,6 @@ import { Sceanario } from '../../types';
 
 export const data: Sceanario[] = [
   {
-    output: [1, 2, 3, ['string1', 0.2], ['string2', 'string3', 'aa"a', true, false], 2],
+    output: [1, 2, 3, ['string1', 20.02], ['string2', 'string3', 'aa"a', true, false], 2],
   },
 ];
diff --git a/test/scenarios/arrays/template.jt b/test/scenarios/arrays/template.jt
@@ -1,6 +1,6 @@
 let a = [
     "string1", `string2`, 'string3', "aa\"a", true, false, 
-    undefined, null, .2, 0.22, [1, 2, 3], {"b": [1, 2]}
+    undefined, null, 20.02, .22, [1., 2, 3], {"b": [1, 2]}
 ];
 [...a[-2], a[0,8], a[1:6], a[-1].b[1]]
 
diff --git a/test/scenarios/compile_time_expressions/data.ts b/test/scenarios/compile_time_expressions/data.ts
@@ -2,7 +2,7 @@ import { Sceanario } from '../../types';
 
 export const data: Sceanario[] = [
   {
-    compileTimeBindings: { a: 1, b: 'string', c: { c: 1 }, d: [null, true, false] },
-    output: [1, 'string', { c: 1 }, [null, true, false]],
+    compileTimeBindings: { a: 1, b: 'string', c: { c: 1.02 }, d: [null, true, false] },
+    output: [1, 'string', { c: 1.02 }, [null, true, false]],
   },
 ];
diff --git a/test/scenarios/paths/data.ts b/test/scenarios/paths/data.ts
@@ -4,19 +4,24 @@ export const data: Sceanario[] = [
   {
     input: [
       {
-        a: { d: 1 },
-        b: [{ d: 2 }, { d: 3 }],
-        c: { c: { d: 4 } },
+        a: { e: 1 },
+        b: [{ e: 2 }, { e: [3, 2] }],
+        c: { c: { e: 4 } },
+        d: [
+          [1, 2],
+          [3, 4],
+        ],
       },
     ],
     output: [
       [3],
       'aa',
       {
-        d: 1,
+        e: 1,
       },
       4,
       3,
+      4,
       1,
     ],
   },

diff --git a/test/scenarios/paths/template.jt b/test/scenarios/paths/template.jt
@@ -1,8 +1,9 @@
 [
     3[],
     "aa"[0][][0][][0],
-    ^.d ?? ^.a,
-    ^.c.d ?? .c.c.d,
-    .b[1].d ?? .b[0].d,
+    ^.e ?? ^.a,
+    ^.c.e ?? .c.c.e,
+    .b.1.e.0 ?? .b.0.e,
+    .d.1.1,
     ^[].length
 ]
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,7 +15,8 @@ export enum Keyword { @@
     export enum TokenType {
       UNKNOWN,
       ID,
-      NUM,
+      INT,
+      FLOAT,
       STR,
       BOOL,
       NULL,
@@ Expand Down @@