Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unify the blockquote parsing logic for messages and rawInput #820

Merged
merged 8 commits into from
Dec 5, 2024
Merged

Unify the blockquote parsing logic for messages and rawInput #820

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6e3bae9
Unify the blockquote parsing for shouldKeepRawInput = false
Skalakid Nov 15, 2024
55a2051
Remove duplicated code
Skalakid Nov 15, 2024
4665020
Fix tests
Skalakid Nov 18, 2024
a1511ab
Fix HTML to markdown parsing
Skalakid Nov 22, 2024
e93727f
Fix html to markdown tests
Skalakid Nov 22, 2024
7633d6d
Add function description
Skalakid Nov 25, 2024
d12e9ed
Add review changes
Skalakid Dec 4, 2024
6611dab
Remove unused code
Skalakid Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 16 additions & 22 deletions __tests__/ExpensiMark-HTML-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1125,13 +1125,13 @@ test('Test quotes markdown replacement and removing <br/> from <br/><pre> and </

test('Test quotes markdown replacement skipping blank quotes', () => {
const testString = '> \n>';
const resultString = '&gt; <br />&gt;';
const resultString = '<blockquote> </blockquote>&gt;';
expect(parser.replace(testString)).toBe(resultString);
});

test('Test quotes markdown replacement with text starts with blank quote', () => {
const testString = '> \ntest';
const resultString = '&gt; <br />test';
const resultString = '<blockquote> </blockquote>test';
expect(parser.replace(testString)).toBe(resultString);
});

Expand All @@ -1143,7 +1143,7 @@ test('Test quotes markdown replacement with quotes starts with blank quote row',

test('Test quotes markdown replacement with quotes ends with blank quote rows', () => {
const testString = '> test\n> \n>';
const resultString = '<blockquote>test<br /> <br /> </blockquote>';
const resultString = '<blockquote>test<br /> </blockquote>&gt;';
expect(parser.replace(testString)).toBe(resultString);
});

Expand All @@ -1162,14 +1162,14 @@ test('Test quotes markdown replacement with quotes includes multiple middle blan
test('Test quotes markdown replacement with text includes blank quotes', () => {
const testString = '> \n> quote1 line a\n> quote1 line b\ntest\n> \ntest\n> quote2 line a\n> \n> \n> quote2 line b with an empty line above';
const resultString =
'<blockquote> <br />quote1 line a<br />quote1 line b</blockquote>test<br />&gt; <br />test<br /><blockquote>quote2 line a<br /> <br /> <br />quote2 line b with an empty line above</blockquote>';
'<blockquote> <br />quote1 line a<br />quote1 line b</blockquote>test<br /><blockquote> </blockquote>test<br /><blockquote>quote2 line a<br /> <br /> <br />quote2 line b with an empty line above</blockquote>';
expect(parser.replace(testString)).toBe(resultString);
});

test('Test quotes markdown replacement with text includes multiple spaces', () => {
const quoteTestStartString = '> Indented\n>No indent\n> Indented \n> > Nested indented \n> Indented ';
const quoteTestStartString = '> Indented\n>No indent\n> Indented \n>> Nested indented \n> > Nested not indented \n> Indented ';
const quoteTestReplacedString =
'<blockquote> Indented</blockquote>&gt;No indent<br /><blockquote> Indented <br /><blockquote> Nested indented </blockquote> Indented </blockquote>';
'<blockquote> Indented</blockquote>&gt;No indent<br /><blockquote> Indented <br /><blockquote> Nested indented </blockquote>&gt; Nested not indented <br /> Indented </blockquote>';
expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});

Expand All @@ -1192,13 +1192,7 @@ test('Test markdown quotes without spaces after > should not be parsed', () => {

test('Test markdown quotes without spaces after > should not be parsed', () => {
const testString = '> > > test';
const resultString = '<blockquote><blockquote><blockquote>test</blockquote></blockquote></blockquote>';
expect(parser.replace(testString)).toBe(resultString);
});

test('Test markdown quotes without spaces after > should not be parsed', () => {
const testString = '>>> test';
const resultString = '&gt;&gt;&gt; test';
const resultString = '<blockquote>&gt; &gt; test</blockquote>';
expect(parser.replace(testString)).toBe(resultString);
});

Expand Down Expand Up @@ -2027,54 +2021,54 @@ test('Test italic/bold/strikethrough markdown to keep consistency', () => {

describe('multi-level blockquote', () => {
test('test max level of blockquote (3)', () => {
const quoteTestStartString = '> > > > > Hello world';
const quoteTestReplacedString = '<blockquote><blockquote><blockquote>&gt; &gt; Hello world</blockquote></blockquote></blockquote>';
const quoteTestStartString = '>>>>> Hello world';
const quoteTestReplacedString = '<blockquote><blockquote><blockquote>&gt;&gt; Hello world</blockquote></blockquote></blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});
test('multi-level blockquote with single space', () => {
const quoteTestStartString = '> > > Hello world';
const quoteTestStartString = '>>> Hello world';
const quoteTestReplacedString = '<blockquote><blockquote><blockquote>Hello world</blockquote></blockquote></blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});
test('multi-level blockquote with multiple spaces', () => {
const quoteTestStartString = '> > > Hello world';
const quoteTestReplacedString = '<blockquote><blockquote><blockquote> Hello world</blockquote></blockquote></blockquote>';
const quoteTestReplacedString = '<blockquote> &gt; &gt; Hello world</blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});

test('multi-level blockquote with mixed spaces', () => {
const quoteTestStartString = '> > > Hello world';
const quoteTestReplacedString = '<blockquote><blockquote><blockquote> Hello world</blockquote></blockquote></blockquote>';
const quoteTestReplacedString = '<blockquote> &gt; &gt; Hello world</blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});

test('multi-level blockquote with diffrent syntax', () => {
const quoteTestStartString = '> > _Hello_ *world*';
const quoteTestStartString = '>> _Hello_ *world*';
const quoteTestReplacedString = '<blockquote><blockquote><em>Hello</em> <strong>world</strong></blockquote></blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});

test('multi-level blockquote with nested heading', () => {
const quoteTestStartString = '> > # Hello world';
const quoteTestStartString = '>> # Hello world';
const quoteTestReplacedString = '<blockquote><blockquote><h1>Hello world</h1></blockquote></blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});

test('multiline multi-level blockquote', () => {
const quoteTestStartString = '> > Hello my\n> > beautiful\n> > world\n';
const quoteTestStartString = '>> Hello my\n>> beautiful\n>> world\n';
const quoteTestReplacedString = '<blockquote><blockquote>Hello my<br />beautiful<br />world</blockquote></blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
});

test('multiline blockquote with diffrent levels', () => {
const quoteTestStartString = '> > > Hello my\n> > beautiful\n> world\n';
const quoteTestStartString = '>>> Hello my\n>> beautiful\n> world\n';
const quoteTestReplacedString = '<blockquote><blockquote><blockquote>Hello my</blockquote>beautiful</blockquote>world</blockquote>';

expect(parser.replace(quoteTestStartString)).toBe(quoteTestReplacedString);
Expand Down
129 changes: 82 additions & 47 deletions lib/ExpensiMark.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,58 +421,26 @@ export default class ExpensiMark {
// block quotes naturally appear on their own line. Blockquotes should not appear in code fences or
// inline code blocks. A single prepending space should be stripped if it exists
process: (textToProcess, replacement, shouldKeepRawInput = false) => {
const regex = /^(?:&gt;)+ +(?! )(?![^<]*(?:<\/pre>|<\/code>|<\/video>))([^\v\n\r]+)/gm;
const regex = /^(?:&gt;)+ +(?! )(?![^<]*(?:<\/pre>|<\/code>|<\/video>))([^\v\n\r]*)/gm;

let replacedText = this.replaceTextWithExtras(textToProcess, regex, EXTRAS_DEFAULT, replacement);
if (shouldKeepRawInput) {
const rawInputRegex = /^(?:&gt;)+ +(?! )(?![^<]*(?:<\/pre>|<\/code>|<\/video>))([^\v\n\r]*)/gm;
return this.replaceTextWithExtras(textToProcess, rawInputRegex, EXTRAS_DEFAULT, replacement);
return replacedText;
}

for (let i = this.maxQuoteDepth; i > 0; i--) {
replacedText = replacedText.replaceAll(`${'</blockquote>'.repeat(i)}\n${'<blockquote>'.repeat(i)}`, '\n');
}
return this.modifyTextForQuote(regex, textToProcess, replacement as ReplacementFn);
replacedText = replacedText.replaceAll('</blockquote>\n', '</blockquote>');
return replacedText;
},
replacement: (_extras, g1) => {
// We want to enable 2 options of nested heading inside the blockquote: "># heading" and "> # heading".
// To do this we need to parse body of the quote without first space
const handleMatch = (match: string) => match;
const textToReplace = g1.replace(/^&gt;( )?/gm, handleMatch);
const filterRules = ['heading1'];

// if we don't reach the max quote depth we allow the recursive call to process possible quote
if (this.currentQuoteDepth < this.maxQuoteDepth - 1) {
filterRules.push('quote');
this.currentQuoteDepth++;
}

const replacedText = this.replace(textToReplace, {
filterRules,
shouldEscapeText: false,
shouldKeepRawInput: false,
});
this.currentQuoteDepth = 0;
return `<blockquote>${replacedText}</blockquote>`;
const {replacedText} = this.replaceQuoteText(g1, false);
return `<blockquote>${replacedText || ' '}</blockquote>`;
},
rawInputReplacement: (_extras, g1) => {
// We want to enable 2 options of nested heading inside the blockquote: "># heading" and "> # heading".
// To do this we need to parse body of the quote without first space
let isStartingWithSpace = false;
const handleMatch = (_match: string, g2: string) => {
isStartingWithSpace = !!g2;
return '';
};
const textToReplace = g1.replace(/^&gt;( )?/gm, handleMatch);
const filterRules = ['heading1'];

// if we don't reach the max quote depth we allow the recursive call to process possible quote
if (this.currentQuoteDepth < this.maxQuoteDepth - 1 || isStartingWithSpace) {
filterRules.push('quote');
this.currentQuoteDepth++;
}

const replacedText = this.replace(textToReplace, {
filterRules,
shouldEscapeText: false,
shouldKeepRawInput: true,
});
this.currentQuoteDepth = 0;
return `<blockquote>${isStartingWithSpace ? ' ' : ''}${replacedText}</blockquote>`;
const {replacedText, shouldAddSpace} = this.replaceQuoteText(g1, true);
return `<blockquote>${shouldAddSpace ? ' ' : ''}${replacedText}</blockquote>`;
},
},
/**
Expand Down Expand Up @@ -1129,8 +1097,9 @@ export default class ExpensiMark {
return;
}

const nextItem = splitText?.[index + 1];
// Insert '\n' unless it ends with '\n' or '>' or it's the last element, or if it's a header ('# ') with a space.
if (text.match(/[\n|>][>]?[\s]?$/) || index === splitText.length - 1 || text === '# ') {
if ((nextItem && text.match(/>[\s]?$/) && !nextItem.startsWith('> ')) || text.match(/\n[\s]?$/) || index === splitText.length - 1 || text === '# ') {
joinedText += text;
} else {
joinedText += `${text}\n`;
Expand All @@ -1142,6 +1111,48 @@ export default class ExpensiMark {
return joinedText;
}

/**
* Unpacks nested quotes HTML tags that have been packed by the 'quote' rule in this.rules for shouldKeepRawInput = false
Skalakid marked this conversation as resolved.
Show resolved Hide resolved
*/
unpackNestedQuotes(text: string): string {
let parsedText = text.replace(/((<\/blockquote>)+(<br \/>)?)|(<br \/>)/g, (match) => {
return `${match}</split>`;
});
const splittedText = parsedText.split('</split>');
if (splittedText.length > 0 && splittedText[splittedText.length - 1] === '') {
splittedText.pop();
}

let count = 0;
parsedText = splittedText
.map((line) => {
const hasBR = line.endsWith('<br />');
if (line === '' && count === 0) {
return '';
}
const textLine = line.replace(/(<br \/>)$/g, '');

if (textLine.startsWith('<blockquote>')) {
count += (textLine.match(/<blockquote>/g) || []).length;
}
if (textLine.endsWith('</blockquote>')) {
count -= (textLine.match(/<\/blockquote>/g) || []).length;
if (count > 0) {
return `${textLine}${'<blockquote>'.repeat(count)}`;
}
}
Skalakid marked this conversation as resolved.
Show resolved Hide resolved

if (count > 0) {
return `${textLine}${'</blockquote>'}${'<blockquote>'.repeat(count)}`;
}

return textLine + (hasBR ? '<br />' : '');
})
.join('');

return parsedText;
}

/**
* Replaces HTML with markdown
*/
Expand All @@ -1154,6 +1165,7 @@ export default class ExpensiMark {
if (parseBodyTag) {
generatedMarkdown = parseBodyTag[2];
}
generatedMarkdown = this.unpackNestedQuotes(generatedMarkdown);

const processRule = (rule: RuleWithRegex) => {
// Pre-processes input HTML before applying regex
Expand Down Expand Up @@ -1273,6 +1285,29 @@ export default class ExpensiMark {
return textToCheck;
}

replaceQuoteText(text: string, shouldKeepRawInput: boolean): {replacedText: string; shouldAddSpace: boolean} {
pecanoro marked this conversation as resolved.
Show resolved Hide resolved
let isStartingWithSpace = false;
const handleMatch = (_match: string, g2: string) => {
isStartingWithSpace = !!g2;
return '';
};
const textToReplace = text.replace(/^&gt;( )?/gm, handleMatch);
const filterRules = ['heading1'];
// if we don't reach the max quote depth we allow the recursive call to process possible quote
Skalakid marked this conversation as resolved.
Show resolved Hide resolved
if (this.currentQuoteDepth < this.maxQuoteDepth - 1 && !isStartingWithSpace) {
filterRules.push('quote');
this.currentQuoteDepth++;
}
const replacedText = this.replace(textToReplace, {
filterRules,
shouldEscapeText: false,
shouldKeepRawInput,
});
this.currentQuoteDepth = 0;

return {replacedText, shouldAddSpace: isStartingWithSpace};
}

/**
* Check if the input text includes only the open or the close tag of an element.
*/
Expand Down
Loading