Refactor and Example for LexerLess Parser (ECMAScript) (#531)

* WIP - ScannerLess Parser example. fixes #528 * Refactor the Parser to allow implementing a simple LexerLess parser by using method overrides. A more proper solution with depedency injection proved too damaging to performance... Part of #521 Fixes #528
Chevrotain · Jul 25, 2017 · 6fa65ae · 6fa65ae
1 parent eeab341
commit 6fa65ae
Show file tree

Hide file tree

Showing 5 changed files with 494 additions and 239 deletions.
diff --git a/src/parse/grammar/lookahead.ts b/src/parse/grammar/lookahead.ts
@@ -16,7 +16,8 @@ import {
  IAnyOrAlt,
  TokenMatcher,
  TokenInstanceIdentityFunc,
- TokenClassIdentityFunc
+ TokenClassIdentityFunc,
+ lookAheadSequence
 } from "../parser_public"
 import { TokenConstructor } from "../../scan/lexer_public"
 
@@ -55,10 +56,12 @@ export function buildLookaheadFuncForOr(
  tokenMatcher: TokenMatcher,
  tokenClassIdentityFunc: TokenClassIdentityFunc,
  tokenIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
+ dynamicTokensEnabled: boolean,
+ laFuncBuilder: Function
 ): (orAlts?: IAnyOrAlt<any>[]) => number {
  let lookAheadPaths = getLookaheadPathsForOr(occurrence, ruleGrammar, k)
- return buildAlternativesLookAheadFunc(
+
+ return laFuncBuilder(
  lookAheadPaths,
  hasPredicates,
  tokenMatcher,
@@ -83,126 +86,29 @@ export function buildLookaheadFuncForOr(
 export function buildLookaheadFuncForOptionalProd(
  occurrence: number,
  ruleGrammar: gast.Rule,
- prodType: PROD_TYPE,
  k: number,
  tokenMatcher: TokenMatcher,
  tokenClassIdentityFunc: TokenClassIdentityFunc,
  tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
+ dynamicTokensEnabled: boolean,
+ prodType: PROD_TYPE,
+ lookaheadBuilder: (
+ lookAheadSequence,
+ TokenMatcher,
+ TokenClassIdentityFunc,
+ TokenInstanceIdentityFunc,
+ boolean
+ ) => () => boolean
 ): () => boolean {
  let lookAheadPaths = getLookaheadPathsForOptionalProd(
  occurrence,
  ruleGrammar,
  prodType,
  k
  )
- return buildSingleAlternativeLookaheadFunction(
- lookAheadPaths[0],
- tokenMatcher,
- tokenClassIdentityFunc,
- tokenInstanceIdentityFunc,
- dynamicTokensEnabled
- )
-}
-
-export function buildLookaheadForOption(
- optionOccurrence: number,
- ruleGrammar: gast.Rule,
- k: number,
- tokenMatcher: TokenMatcher,
- tokenClassIdentityFunc: TokenClassIdentityFunc,
- tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
-): () => boolean {
- return buildLookaheadFuncForOptionalProd(
- optionOccurrence,
- ruleGrammar,
- PROD_TYPE.OPTION,
- k,
- tokenMatcher,
- tokenClassIdentityFunc,
- tokenInstanceIdentityFunc,
- dynamicTokensEnabled
- )
-}
-
-export function buildLookaheadForMany(
- optionOccurrence: number,
- ruleGrammar: gast.Rule,
- k: number,
- tokenMatcher: TokenMatcher,
- tokenClassIdentityFunc: TokenClassIdentityFunc,
- tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
-): () => boolean {
- return buildLookaheadFuncForOptionalProd(
- optionOccurrence,
- ruleGrammar,
- PROD_TYPE.REPETITION,
- k,
- tokenMatcher,
- tokenClassIdentityFunc,
- tokenInstanceIdentityFunc,
- dynamicTokensEnabled
- )
-}
 
-export function buildLookaheadForManySep(
- optionOccurrence: number,
- ruleGrammar: gast.Rule,
- k: number,
- tokenMatcher: TokenMatcher,
- tokenClassIdentityFunc: TokenClassIdentityFunc,
- tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
-): () => boolean {
- return buildLookaheadFuncForOptionalProd(
- optionOccurrence,
- ruleGrammar,
- PROD_TYPE.REPETITION_WITH_SEPARATOR,
- k,
- tokenMatcher,
- tokenClassIdentityFunc,
- tokenInstanceIdentityFunc,
- dynamicTokensEnabled
- )
-}
-
-export function buildLookaheadForAtLeastOne(
- optionOccurrence: number,
- ruleGrammar: gast.Rule,
- k: number,
- tokenMatcher: TokenMatcher,
- tokenIdentityFunc: TokenClassIdentityFunc,
- tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
-): () => boolean {
- return buildLookaheadFuncForOptionalProd(
- optionOccurrence,
- ruleGrammar,
- PROD_TYPE.REPETITION_MANDATORY,
- k,
- tokenMatcher,
- tokenIdentityFunc,
- tokenInstanceIdentityFunc,
- dynamicTokensEnabled
- )
-}
-
-export function buildLookaheadForAtLeastOneSep(
- optionOccurrence: number,
- ruleGrammar: gast.Rule,
- k: number,
- tokenMatcher: TokenMatcher,
- tokenClassIdentityFunc: TokenClassIdentityFunc,
- tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
- dynamicTokensEnabled: boolean
-): () => boolean {
- return buildLookaheadFuncForOptionalProd(
- optionOccurrence,
- ruleGrammar,
- PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
- k,
+ return lookaheadBuilder(
+ lookAheadPaths[0],
  tokenMatcher,
  tokenClassIdentityFunc,
  tokenInstanceIdentityFunc,
@@ -211,7 +117,6 @@ export function buildLookaheadForAtLeastOneSep(
 }
 
 export type Alternative = TokenConstructor[][]
-export type lookAheadSequence = TokenConstructor[][]
 
 export function buildAlternativesLookAheadFunc(
  alts: lookAheadSequence[],
@@ -387,10 +292,9 @@ export function buildSingleAlternativeLookaheadFunction(
  )
  return function(): boolean {
  let nextToken = this.LA(1)
- return choiceToAlt[tokenInstanceIdentityFunc(nextToken)] ===
- true
- ? true
- : false
+ return (
+ choiceToAlt[tokenInstanceIdentityFunc(nextToken)] === true
+ )
  }
  }
  } else {