Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix equation parsing issues #283

Merged
merged 1 commit into from
Mar 28, 2018
Merged

fix equation parsing issues #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 39 additions & 4 deletions packages/idyll-compiler/src/lexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@ const formatToken = (text) => {
let currentInput = null;

const lex = function(options) {
let { row, column, outer, skipLists } = Object.assign({}, {
let { row, column, outer, skipLists, inComponent, gotName } = Object.assign({}, {
row: 1,
column: 1,
outer: true,
skipLists: false
skipLists: false,
inComponent: false,
gotName: false
}, options || {});
var lexer = new Lexer(function (chr) {
let errorString = `
Expand All @@ -29,8 +31,6 @@ const lex = function(options) {
`
throw new Error(errorString);
});
var inComponent = false;
var gotName = false;

const recurse = (str, opts) => {
return lex(Object.assign({ row, column, outer: false }, opts || {}))(str).tokens;
Expand All @@ -45,6 +45,39 @@ const lex = function(options) {
column += lines[lines.length - 1].length;
}


// Rules at the front are pre-processed,
// e.g. equations, and code snippets
// that shouldn't be formatted.

lexer.addRule(/\[\s*equation\s*(((`[^`]*`)|([^`\]]*))*)\s*\](((?!(\[\s*equation\s*\])).)*)\[\s*\/\s*equation\s*\]/i, function(lexeme, props, _1, _2, _3, innerText) {
inComponent = true;
if (this.reject) return;
updatePosition(lexeme);
return ['OPEN_BRACKET', 'COMPONENT_NAME']
.concat(formatToken('equation'))
.concat(recurse(props, { inComponent: true, gotName: true }))
.concat(['CLOSE_BRACKET'])
.concat(['WORDS'])
.concat(formatToken(innerText))
.concat(['OPEN_BRACKET', 'FORWARD_SLASH', 'COMPONENT_NAME'])
.concat(formatToken('equation'))
.concat(['CLOSE_BRACKET']);
});
lexer.addRule(/\[\s*code\s*(((`[^`]*`)|([^`\]]*))*)\s*\](((?!(\[\s*code\s*\])).)*)\[\s*\/\s*code\s*\]/i, function(lexeme, props, _1, _2, _3, innerText) {
inComponent = true;
if (this.reject) return;
updatePosition(lexeme);
return ['OPEN_BRACKET', 'COMPONENT_NAME']
.concat(formatToken('code'))
.concat(recurse(props, { inComponent: true, gotName: true }))
.concat(['CLOSE_BRACKET'])
.concat(['WORDS'])
.concat(formatToken(innerText))
.concat(['OPEN_BRACKET', 'FORWARD_SLASH', 'COMPONENT_NAME'])
.concat(formatToken('code'))
.concat(['CLOSE_BRACKET']);
});
lexer.addRule(/`{4}(\S*)\n(((?!````)[\s\S])+)`{4}/g, function(lexeme, language, text) {
this.reject = inComponent;
if (this.reject) return;
Expand Down Expand Up @@ -208,6 +241,8 @@ const lex = function(options) {
updatePosition(lexeme);
});



lexer.addRule(/\[/, function(lexeme) {
inComponent = true;
if (this.reject) return;
Expand Down
1 change: 0 additions & 1 deletion packages/idyll-compiler/src/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ module.exports = function(input, tokens, positions, options) {
options = options || {};

const p = new nearley.Parser(grammar.ParserRules, grammar.ParserStart);

try {
p.feed(tokens);
} catch(err) {
Expand Down
56 changes: 41 additions & 15 deletions packages/idyll-compiler/test/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ describe('compiler', function() {
it('should handle equations', function () {
var lex = Lexer();
var results = lex("[Equation]y = 0[/Equation]");
expect(results.tokens.join(' ')).to.eql('OPEN_BRACKET COMPONENT_NAME TOKEN_VALUE_START "Equation" TOKEN_VALUE_END CLOSE_BRACKET WORDS TOKEN_VALUE_START "y = " TOKEN_VALUE_END WORDS TOKEN_VALUE_START "0" TOKEN_VALUE_END OPEN_BRACKET FORWARD_SLASH COMPONENT_NAME TOKEN_VALUE_START "Equation" TOKEN_VALUE_END CLOSE_BRACKET EOF');
expect(results.tokens.join(' ')).to.eql('OPEN_BRACKET COMPONENT_NAME TOKEN_VALUE_START "equation" TOKEN_VALUE_END CLOSE_BRACKET WORDS TOKEN_VALUE_START "y = 0" TOKEN_VALUE_END OPEN_BRACKET FORWARD_SLASH COMPONENT_NAME TOKEN_VALUE_START "equation" TOKEN_VALUE_END CLOSE_BRACKET EOF');
});

it('should handle backticks in a paragraph', function() {
Expand Down Expand Up @@ -528,7 +528,7 @@ End text
const input = "[Equation]y = 0[/Equation]";
expect(compile(input)).to.eql([
['TextContainer', [], [
['Equation', [], ['y = 0']]
['equation', [], ['y = 0']]
]]
]);
})
Expand Down Expand Up @@ -663,21 +663,47 @@ End text
]);
});

// it('should handle equations with strange things inside', function() {
it('should handle equations with strange things inside - 1', function() {

// const input = `[Equation display:\`true\`] \\sum_{j=0}^n x^{j} + \\sum x^{k} [/Equation]`;
const input = `[equation display:true]\sum_{j=0}^n x^{j} + \sum x^{k}[/equation]`;

// expect(compile(input)).to.eql(
// [
// ['TextContainer', [], [
// ['p', [], [
// ['em', [], ['text']],
// ' ',
// ['b', [], ['other text']]
// ]]
// ]]
// ]);
// });

expect(compile(input)).to.eql(
[
['TextContainer', [], [
['equation', [['display', ['value', true]]], [
'\sum_{j=0}^n x^{j} + \sum x^{k}'
]]
]]
]);
});

it('should handle equations with strange things inside - 2', function() {

const input = `[equation display:true]\sum_{j=0}^n x^{j} + \sum_{k=0}^n x^{k}[/equation]`;


expect(compile(input)).to.eql(
[
['TextContainer', [], [
['equation', [['display', ['value', true]]], [
'\sum_{j=0}^n x^{j} + \sum_{k=0}^n x^{k}'
]]
]]
]);
});

it('should handle code blocks with parens inside', function() {
const input = `[code](n - 1)!/2 possible paths[/code]`;
expect(compile(input)).to.eql(
[
['TextContainer', [], [
['code', [], [
'(n - 1)!/2 possible paths'
]]
]]
]);
});

});

Expand Down