diff --git a/packages/idyll-compiler/src/lexer.js b/packages/idyll-compiler/src/lexer.js index 8715fb3e8..0444a210e 100644 --- a/packages/idyll-compiler/src/lexer.js +++ b/packages/idyll-compiler/src/lexer.js @@ -13,11 +13,13 @@ const formatToken = (text) => { let currentInput = null; const lex = function(options) { - let { row, column, outer, skipLists } = Object.assign({}, { + let { row, column, outer, skipLists, inComponent, gotName } = Object.assign({}, { row: 1, column: 1, outer: true, - skipLists: false + skipLists: false, + inComponent: false, + gotName: false }, options || {}); var lexer = new Lexer(function (chr) { let errorString = ` @@ -29,8 +31,6 @@ const lex = function(options) { ` throw new Error(errorString); }); - var inComponent = false; - var gotName = false; const recurse = (str, opts) => { return lex(Object.assign({ row, column, outer: false }, opts || {}))(str).tokens; @@ -45,6 +45,39 @@ const lex = function(options) { column += lines[lines.length - 1].length; } + + // Rules at the front are pre-processed, + // e.g. equations, and code snippets + // that shouldn't be formatted. + + lexer.addRule(/\[\s*equation\s*(((`[^`]*`)|([^`\]]*))*)\s*\](((?!(\[\s*equation\s*\])).)*)\[\s*\/\s*equation\s*\]/i, function(lexeme, props, _1, _2, _3, innerText) { + inComponent = true; + if (this.reject) return; + updatePosition(lexeme); + return ['OPEN_BRACKET', 'COMPONENT_NAME'] + .concat(formatToken('equation')) + .concat(recurse(props, { inComponent: true, gotName: true })) + .concat(['CLOSE_BRACKET']) + .concat(['WORDS']) + .concat(formatToken(innerText)) + .concat(['OPEN_BRACKET', 'FORWARD_SLASH', 'COMPONENT_NAME']) + .concat(formatToken('equation')) + .concat(['CLOSE_BRACKET']); + }); + lexer.addRule(/\[\s*code\s*(((`[^`]*`)|([^`\]]*))*)\s*\](((?!(\[\s*code\s*\])).)*)\[\s*\/\s*code\s*\]/i, function(lexeme, props, _1, _2, _3, innerText) { + inComponent = true; + if (this.reject) return; + updatePosition(lexeme); + return ['OPEN_BRACKET', 'COMPONENT_NAME'] + .concat(formatToken('code')) + .concat(recurse(props, { inComponent: true, gotName: true })) + .concat(['CLOSE_BRACKET']) + .concat(['WORDS']) + .concat(formatToken(innerText)) + .concat(['OPEN_BRACKET', 'FORWARD_SLASH', 'COMPONENT_NAME']) + .concat(formatToken('code')) + .concat(['CLOSE_BRACKET']); + }); lexer.addRule(/`{4}(\S*)\n(((?!````)[\s\S])+)`{4}/g, function(lexeme, language, text) { this.reject = inComponent; if (this.reject) return; @@ -208,6 +241,8 @@ const lex = function(options) { updatePosition(lexeme); }); + + lexer.addRule(/\[/, function(lexeme) { inComponent = true; if (this.reject) return; diff --git a/packages/idyll-compiler/src/parser.js b/packages/idyll-compiler/src/parser.js index 650e763f0..2bf5c595a 100644 --- a/packages/idyll-compiler/src/parser.js +++ b/packages/idyll-compiler/src/parser.js @@ -5,7 +5,6 @@ module.exports = function(input, tokens, positions, options) { options = options || {}; const p = new nearley.Parser(grammar.ParserRules, grammar.ParserStart); - try { p.feed(tokens); } catch(err) { diff --git a/packages/idyll-compiler/test/test.js b/packages/idyll-compiler/test/test.js index 609be8e26..1a3517bcd 100644 --- a/packages/idyll-compiler/test/test.js +++ b/packages/idyll-compiler/test/test.js @@ -45,7 +45,7 @@ describe('compiler', function() { it('should handle equations', function () { var lex = Lexer(); var results = lex("[Equation]y = 0[/Equation]"); - expect(results.tokens.join(' ')).to.eql('OPEN_BRACKET COMPONENT_NAME TOKEN_VALUE_START "Equation" TOKEN_VALUE_END CLOSE_BRACKET WORDS TOKEN_VALUE_START "y = " TOKEN_VALUE_END WORDS TOKEN_VALUE_START "0" TOKEN_VALUE_END OPEN_BRACKET FORWARD_SLASH COMPONENT_NAME TOKEN_VALUE_START "Equation" TOKEN_VALUE_END CLOSE_BRACKET EOF'); + expect(results.tokens.join(' ')).to.eql('OPEN_BRACKET COMPONENT_NAME TOKEN_VALUE_START "equation" TOKEN_VALUE_END CLOSE_BRACKET WORDS TOKEN_VALUE_START "y = 0" TOKEN_VALUE_END OPEN_BRACKET FORWARD_SLASH COMPONENT_NAME TOKEN_VALUE_START "equation" TOKEN_VALUE_END CLOSE_BRACKET EOF'); }); it('should handle backticks in a paragraph', function() { @@ -528,7 +528,7 @@ End text const input = "[Equation]y = 0[/Equation]"; expect(compile(input)).to.eql([ ['TextContainer', [], [ - ['Equation', [], ['y = 0']] + ['equation', [], ['y = 0']] ]] ]); }) @@ -663,21 +663,47 @@ End text ]); }); - // it('should handle equations with strange things inside', function() { + it('should handle equations with strange things inside - 1', function() { - // const input = `[Equation display:\`true\`] \\sum_{j=0}^n x^{j} + \\sum x^{k} [/Equation]`; + const input = `[equation display:true]\sum_{j=0}^n x^{j} + \sum x^{k}[/equation]`; - // expect(compile(input)).to.eql( - // [ - // ['TextContainer', [], [ - // ['p', [], [ - // ['em', [], ['text']], - // ' ', - // ['b', [], ['other text']] - // ]] - // ]] - // ]); - // }); + + expect(compile(input)).to.eql( + [ + ['TextContainer', [], [ + ['equation', [['display', ['value', true]]], [ + '\sum_{j=0}^n x^{j} + \sum x^{k}' + ]] + ]] + ]); + }); + + it('should handle equations with strange things inside - 2', function() { + + const input = `[equation display:true]\sum_{j=0}^n x^{j} + \sum_{k=0}^n x^{k}[/equation]`; + + + expect(compile(input)).to.eql( + [ + ['TextContainer', [], [ + ['equation', [['display', ['value', true]]], [ + '\sum_{j=0}^n x^{j} + \sum_{k=0}^n x^{k}' + ]] + ]] + ]); + }); + + it('should handle code blocks with parens inside', function() { + const input = `[code](n - 1)!/2 possible paths[/code]`; + expect(compile(input)).to.eql( + [ + ['TextContainer', [], [ + ['code', [], [ + '(n - 1)!/2 possible paths' + ]] + ]] + ]); + }); });