From af041c796171acdf2961d7593091d159631d40ac Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Wed, 24 Nov 2021 15:22:43 -0500 Subject: [PATCH 1/8] Fixes applied. Passing all tests, but needs cleanup. --- lib/marked.cjs | 46 +++++++++++++++++------------- lib/marked.esm.js | 44 +++++++++++++++++----------- lib/marked.umd.js | 46 +++++++++++++++++------------- package-lock.json | 6 ++++ src/Tokenizer.js | 41 ++++++++++++++++---------- test/specs/bug/adjacent_lists.html | 4 ++- test/specs/bug/adjacent_lists.md | 3 +- 7 files changed, 116 insertions(+), 74 deletions(-) diff --git a/lib/marked.cjs b/lib/marked.cjs index 06d380c147..7b3b49e544 100644 --- a/lib/marked.cjs +++ b/lib/marked.cjs @@ -533,7 +533,7 @@ var Tokenizer = /*#__PURE__*/function () { var cap = this.rules.block.list.exec(src); if (cap) { - var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, lines, itemContents; + var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, itemContents; var bull = cap[1].trim(); var isordered = bull.length > 1; var list = { @@ -551,7 +551,7 @@ var Tokenizer = /*#__PURE__*/function () { } // Get next list item - var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*| *)(?:\\n[^\\n]*)*(?:\\n|$))"); // Get each top-level item + var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*)?(?:\\n|$))"); // Get each top-level item while (src) { if (this.rules.block.hr.test(src)) { @@ -561,35 +561,41 @@ var Tokenizer = /*#__PURE__*/function () { if (!(cap = itemRegex.exec(src))) { break; - } + } //lines = cap[2].split('\n'); + - lines = cap[2].split('\n'); + raw = cap[0]; + line = cap[2].split('\n', 1)[0]; + src = src.substring(raw.length); + var nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { indent = 2; - itemContents = lines[0].trimLeft(); + itemContents = line.trimLeft(); } else { indent = cap[2].search(/[^ ]/); // Find first non-space char - indent = cap[1].length + (indent > 4 ? 1 : indent); // intented code blocks after 4 spaces; indent is always 1 + indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent - itemContents = lines[0].slice(indent - cap[1].length); + itemContents = line.slice(indent); + indent += cap[1].length; } blankLine = false; - raw = cap[0]; - if (!lines[0] && /^ *$/.test(lines[1])) { + if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line - raw = cap[1] + lines.slice(0, 2).join('\n') + '\n'; + raw += nextLine + '\n'; + src = src.substring(nextLine.length + 1); list.loose = true; - lines = []; } var nextBulletRegex = new RegExp("^ {0," + Math.min(3, indent - 1) + "}(?:[*+-]|\\d{1,9}[.)])"); + var rawLine = void 0; - for (i = 1; i < lines.length; i++) { - line = lines[i]; + while (src && !list.loose) { + rawLine = src.split('\n', 1)[0]; + line = rawLine; if (this.options.pedantic) { // Re-align to follow commonmark nesting rules @@ -598,7 +604,6 @@ var Tokenizer = /*#__PURE__*/function () { if (nextBulletRegex.test(line)) { - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } // Until we encounter a blank line, item contents do not need indentation @@ -616,16 +621,19 @@ var Tokenizer = /*#__PURE__*/function () { itemContents += '\n' + line; } + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } // Dedent this line if (line.search(/[^ ]/) >= indent || !line.trim()) { itemContents += '\n' + line.slice(indent); + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } else { // Line was not properly indented; end of this item - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } } @@ -657,8 +665,7 @@ var Tokenizer = /*#__PURE__*/function () { loose: false, text: itemContents }); - list.raw += raw; - src = src.slice(raw.length); + list.raw += raw; //src = src.slice(raw.length); } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -671,11 +678,10 @@ var Tokenizer = /*#__PURE__*/function () { this.lexer.state.top = false; list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); - if (list.items[i].tokens.some(function (t) { + if (!list.loose && list.items[i].tokens.some(function (t) { return t.type === 'space'; })) { - list.loose = true; - list.items[i].loose = true; + list.loose = true; //list.items[i].loose = true; } } diff --git a/lib/marked.esm.js b/lib/marked.esm.js index 4544a067e0..f961bc8e2d 100644 --- a/lib/marked.esm.js +++ b/lib/marked.esm.js @@ -452,7 +452,7 @@ class Tokenizer { let cap = this.rules.block.list.exec(src); if (cap) { let raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, - line, lines, itemContents; + line, itemContents; let bull = cap[1].trim(); const isordered = bull.length > 1; @@ -473,7 +473,7 @@ class Tokenizer { } // Get next list item - const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*| *)(?:\\n[^\\n]*)*(?:\\n|$))`); + const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*)?(?:\\n|$))`); // Get each top-level item while (src) { @@ -485,30 +485,38 @@ class Tokenizer { break; } - lines = cap[2].split('\n'); + //lines = cap[2].split('\n'); + raw = cap[0]; + line = cap[2].split('\n',1)[0]; + src = src.substring(raw.length); + let nextLine = src.split('\n',1)[0]; if (this.options.pedantic) { indent = 2; - itemContents = lines[0].trimLeft(); + itemContents = line.trimLeft(); } else { indent = cap[2].search(/[^ ]/); // Find first non-space char - indent = cap[1].length + (indent > 4 ? 1 : indent); // intented code blocks after 4 spaces; indent is always 1 - itemContents = lines[0].slice(indent - cap[1].length); + indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent + itemContents = line.slice(indent); + indent += cap[1].length; } blankLine = false; - raw = cap[0]; - if (!lines[0] && /^ *$/.test(lines[1])) { // items begin with at most one blank line - raw = cap[1] + lines.slice(0, 2).join('\n') + '\n'; + + if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line + raw += nextLine + '\n'; + src = src.substring(nextLine.length + 1); list.loose = true; - lines = []; } const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])`); - for (i = 1; i < lines.length; i++) { - line = lines[i]; + let rawLine; + + while(src && !list.loose) { + rawLine = src.split('\n',1)[0]; + line = rawLine; if (this.options.pedantic) { // Re-align to follow commonmark nesting rules line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); @@ -516,7 +524,6 @@ class Tokenizer { // End list item if found start of new bullet if (nextBulletRegex.test(line)) { - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } @@ -532,15 +539,18 @@ class Tokenizer { } else { itemContents += '\n' + line; } + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } // Dedent this line if (line.search(/[^ ]/) >= indent || !line.trim()) { itemContents += '\n' + line.slice(indent); + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } else { // Line was not properly indented; end of this item - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } } @@ -573,7 +583,7 @@ class Tokenizer { }); list.raw += raw; - src = src.slice(raw.length); + //src = src.slice(raw.length); } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -587,9 +597,9 @@ class Tokenizer { for (i = 0; i < l; i++) { this.lexer.state.top = false; list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); - if (list.items[i].tokens.some(t => t.type === 'space')) { + if (!list.loose && list.items[i].tokens.some(t => t.type === 'space')) { list.loose = true; - list.items[i].loose = true; + //list.items[i].loose = true; } } diff --git a/lib/marked.umd.js b/lib/marked.umd.js index 2f2ca9cacb..d571e559e1 100644 --- a/lib/marked.umd.js +++ b/lib/marked.umd.js @@ -535,7 +535,7 @@ var cap = this.rules.block.list.exec(src); if (cap) { - var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, lines, itemContents; + var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, itemContents; var bull = cap[1].trim(); var isordered = bull.length > 1; var list = { @@ -553,7 +553,7 @@ } // Get next list item - var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*| *)(?:\\n[^\\n]*)*(?:\\n|$))"); // Get each top-level item + var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*)?(?:\\n|$))"); // Get each top-level item while (src) { if (this.rules.block.hr.test(src)) { @@ -563,35 +563,41 @@ if (!(cap = itemRegex.exec(src))) { break; - } + } //lines = cap[2].split('\n'); + - lines = cap[2].split('\n'); + raw = cap[0]; + line = cap[2].split('\n', 1)[0]; + src = src.substring(raw.length); + var nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { indent = 2; - itemContents = lines[0].trimLeft(); + itemContents = line.trimLeft(); } else { indent = cap[2].search(/[^ ]/); // Find first non-space char - indent = cap[1].length + (indent > 4 ? 1 : indent); // intented code blocks after 4 spaces; indent is always 1 + indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent - itemContents = lines[0].slice(indent - cap[1].length); + itemContents = line.slice(indent); + indent += cap[1].length; } blankLine = false; - raw = cap[0]; - if (!lines[0] && /^ *$/.test(lines[1])) { + if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line - raw = cap[1] + lines.slice(0, 2).join('\n') + '\n'; + raw += nextLine + '\n'; + src = src.substring(nextLine.length + 1); list.loose = true; - lines = []; } var nextBulletRegex = new RegExp("^ {0," + Math.min(3, indent - 1) + "}(?:[*+-]|\\d{1,9}[.)])"); + var rawLine = void 0; - for (i = 1; i < lines.length; i++) { - line = lines[i]; + while (src && !list.loose) { + rawLine = src.split('\n', 1)[0]; + line = rawLine; if (this.options.pedantic) { // Re-align to follow commonmark nesting rules @@ -600,7 +606,6 @@ if (nextBulletRegex.test(line)) { - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } // Until we encounter a blank line, item contents do not need indentation @@ -618,16 +623,19 @@ itemContents += '\n' + line; } + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } // Dedent this line if (line.search(/[^ ]/) >= indent || !line.trim()) { itemContents += '\n' + line.slice(indent); + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } else { // Line was not properly indented; end of this item - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } } @@ -659,8 +667,7 @@ loose: false, text: itemContents }); - list.raw += raw; - src = src.slice(raw.length); + list.raw += raw; //src = src.slice(raw.length); } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -673,11 +680,10 @@ this.lexer.state.top = false; list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); - if (list.items[i].tokens.some(function (t) { + if (!list.loose && list.items[i].tokens.some(function (t) { return t.type === 'space'; })) { - list.loose = true; - list.items[i].loose = true; + list.loose = true; //list.items[i].loose = true; } } diff --git a/package-lock.json b/package-lock.json index 4a19334c01..d927e18202 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "marked", "version": "4.0.4", "license": "MIT", "bin": { @@ -8362,6 +8363,11 @@ "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" }, + "bin": { + "sshpk-conv": "bin/sshpk-conv", + "sshpk-sign": "bin/sshpk-sign", + "sshpk-verify": "bin/sshpk-verify" + }, "engines": { "node": ">=0.10.0" } diff --git a/src/Tokenizer.js b/src/Tokenizer.js index fe91349c84..54f31e90af 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -190,7 +190,7 @@ export class Tokenizer { } // Get next list item - const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*| *)(?:\\n[^\\n]*)*(?:\\n|$))`); + const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*)?(?:\\n|$))`); // Get each top-level item while (src) { @@ -202,30 +202,39 @@ export class Tokenizer { break; } - lines = cap[2].split('\n'); + //lines = cap[2].split('\n'); + raw = cap[0]; + line = cap[2].split('\n',1)[0]; + src = src.substring(raw.length); + let nextLine = src.split('\n',1)[0]; if (this.options.pedantic) { indent = 2; - itemContents = lines[0].trimLeft(); + itemContents = line.trimLeft(); } else { indent = cap[2].search(/[^ ]/); // Find first non-space char - indent = cap[1].length + (indent > 4 ? 1 : indent); // intented code blocks after 4 spaces; indent is always 1 - itemContents = lines[0].slice(indent - cap[1].length); + indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent + itemContents = line.slice(indent); + indent += cap[1].length; } blankLine = false; - raw = cap[0]; - if (!lines[0] && /^ *$/.test(lines[1])) { // items begin with at most one blank line - raw = cap[1] + lines.slice(0, 2).join('\n') + '\n'; + + if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line + raw += nextLine + '\n'; + src = src.substring(nextLine.length + 1); list.loose = true; lines = []; } const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])`); - for (i = 1; i < lines.length; i++) { - line = lines[i]; + let rawLine; + + while(src && !list.loose) { + rawLine = src.split('\n',1)[0]; + line = rawLine; if (this.options.pedantic) { // Re-align to follow commonmark nesting rules line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); @@ -233,7 +242,6 @@ export class Tokenizer { // End list item if found start of new bullet if (nextBulletRegex.test(line)) { - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } @@ -249,15 +257,18 @@ export class Tokenizer { } else { itemContents += '\n' + line; } + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } // Dedent this line if (line.search(/[^ ]/) >= indent || !line.trim()) { itemContents += '\n' + line.slice(indent); + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); continue; } else { // Line was not properly indented; end of this item - raw = cap[1] + lines.slice(0, i).join('\n') + '\n'; break; } } @@ -290,7 +301,7 @@ export class Tokenizer { }); list.raw += raw; - src = src.slice(raw.length); + //src = src.slice(raw.length); } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -304,9 +315,9 @@ export class Tokenizer { for (i = 0; i < l; i++) { this.lexer.state.top = false; list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); - if (list.items[i].tokens.some(t => t.type === 'space')) { + if (!list.loose && list.items[i].tokens.some(t => t.type === 'space')) { list.loose = true; - list.items[i].loose = true; + //list.items[i].loose = true; } } diff --git a/test/specs/bug/adjacent_lists.html b/test/specs/bug/adjacent_lists.html index 2d64bd5963..0e06965005 100644 --- a/test/specs/bug/adjacent_lists.html +++ b/test/specs/bug/adjacent_lists.html @@ -1 +1,3 @@ -

foo bar *baz bim bam

+ diff --git a/test/specs/bug/adjacent_lists.md b/test/specs/bug/adjacent_lists.md index fdf80c0adb..23e9200a1d 100644 --- a/test/specs/bug/adjacent_lists.md +++ b/test/specs/bug/adjacent_lists.md @@ -1 +1,2 @@ -*foo __bar *baz bim__ bam* +- + foo From f8f02961c1ab0101403e1814bd95adcf0d633099 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Wed, 24 Nov 2021 15:35:08 -0500 Subject: [PATCH 2/8] Remove accidental file. --- test/specs/bug/adjacent_lists.html | 3 --- test/specs/bug/adjacent_lists.md | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 test/specs/bug/adjacent_lists.html delete mode 100644 test/specs/bug/adjacent_lists.md diff --git a/test/specs/bug/adjacent_lists.html b/test/specs/bug/adjacent_lists.html deleted file mode 100644 index 0e06965005..0000000000 --- a/test/specs/bug/adjacent_lists.html +++ /dev/null @@ -1,3 +0,0 @@ - diff --git a/test/specs/bug/adjacent_lists.md b/test/specs/bug/adjacent_lists.md deleted file mode 100644 index 23e9200a1d..0000000000 --- a/test/specs/bug/adjacent_lists.md +++ /dev/null @@ -1,2 +0,0 @@ -- - foo From 802e7eab12750aefe78a1cc13cee99a2403520a4 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Wed, 24 Nov 2021 15:47:31 -0500 Subject: [PATCH 3/8] Cleanup and Lint --- src/Tokenizer.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 54f31e90af..4be825192c 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -169,7 +169,7 @@ export class Tokenizer { let cap = this.rules.block.list.exec(src); if (cap) { let raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, - line, lines, itemContents; + line, nextLine, rawLine, itemContents; let bull = cap[1].trim(); const isordered = bull.length > 1; @@ -192,7 +192,7 @@ export class Tokenizer { // Get next list item const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*)?(?:\\n|$))`); - // Get each top-level item + // Check if current bullet point can start a new List Item while (src) { if (this.rules.block.hr.test(src)) { // End list if we encounter an HR (possibly move into itemRegex?) break; @@ -202,11 +202,10 @@ export class Tokenizer { break; } - //lines = cap[2].split('\n'); raw = cap[0]; - line = cap[2].split('\n',1)[0]; + line = cap[2].split('\n', 1)[0]; src = src.substring(raw.length); - let nextLine = src.split('\n',1)[0]; + nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { indent = 2; @@ -220,20 +219,17 @@ export class Tokenizer { blankLine = false; - if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line raw += nextLine + '\n'; src = src.substring(nextLine.length + 1); list.loose = true; - lines = []; } const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])`); - let rawLine; - - while(src && !list.loose) { - rawLine = src.split('\n',1)[0]; + // Check following lines if they should be included in List Item + while (src && !list.loose) { + rawLine = src.split('\n', 1)[0]; line = rawLine; if (this.options.pedantic) { // Re-align to follow commonmark nesting rules @@ -301,7 +297,6 @@ export class Tokenizer { }); list.raw += raw; - //src = src.slice(raw.length); } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -317,7 +312,7 @@ export class Tokenizer { list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); if (!list.loose && list.items[i].tokens.some(t => t.type === 'space')) { list.loose = true; - //list.items[i].loose = true; + // list.items[i].loose = true; } } From c4744bf29572ccfb7a7e4daa71b00fc5c7b78621 Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Wed, 24 Nov 2021 15:12:08 -0600 Subject: [PATCH 4/8] add quadratic_lists test --- test/specs/redos/quadratic_lists.cjs | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test/specs/redos/quadratic_lists.cjs diff --git a/test/specs/redos/quadratic_lists.cjs b/test/specs/redos/quadratic_lists.cjs new file mode 100644 index 0000000000..8d230ca5af --- /dev/null +++ b/test/specs/redos/quadratic_lists.cjs @@ -0,0 +1,4 @@ +module.exports = { + markdown: '- a\n'.repeat(10000), + html: `
    ${'
  • a
    • '.repeat(10000)}
` +}; From 10b8445bf21d56e61dc9e33fe6405b7a841d67cd Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sat, 27 Nov 2021 16:52:43 -0500 Subject: [PATCH 5/8] Simplify logic to remove redundancy --- src/Tokenizer.js | 48 +++++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 4be825192c..98e79850e0 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -194,17 +194,18 @@ export class Tokenizer { // Check if current bullet point can start a new List Item while (src) { - if (this.rules.block.hr.test(src)) { // End list if we encounter an HR (possibly move into itemRegex?) + if (!(cap = itemRegex.exec(src))) { break; } - if (!(cap = itemRegex.exec(src))) { + if (this.rules.block.hr.test(src)) { // End list if bullet was actually HR (possibly move into itemRegex?) break; } raw = cap[0]; - line = cap[2].split('\n', 1)[0]; src = src.substring(raw.length); + + line = cap[2].split('\n', 1)[0]; nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { @@ -219,7 +220,7 @@ export class Tokenizer { blankLine = false; - if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line + if (!line && /^ *$/.test(nextLine)) { // Items begin with at most one blank line raw += nextLine + '\n'; src = src.substring(nextLine.length + 1); list.loose = true; @@ -227,12 +228,13 @@ export class Tokenizer { const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])`); - // Check following lines if they should be included in List Item + // Check if following lines should be included in List Item while (src && !list.loose) { rawLine = src.split('\n', 1)[0]; line = rawLine; - if (this.options.pedantic) { // Re-align to follow commonmark nesting rules + // Re-align to follow commonmark nesting rules + if (this.options.pedantic) { line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); } @@ -241,32 +243,20 @@ export class Tokenizer { break; } - // Until we encounter a blank line, item contents do not need indentation - if (!blankLine) { - if (!line.trim()) { // Check if current line is empty - blankLine = true; - } - - // Dedent if possible - if (line.search(/[^ ]/) >= indent) { - itemContents += '\n' + line.slice(indent); - } else { - itemContents += '\n' + line; - } - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; - } - - // Dedent this line - if (line.search(/[^ ]/) >= indent || !line.trim()) { + if (line.search(/[^ ]/) >= indent || !line.trim()) { // Dedent if possible itemContents += '\n' + line.slice(indent); - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; - } else { // Line was not properly indented; end of this item + } else if (!blankLine) { // Until blank line, item doesn't need indentation + itemContents += '\n' + line; + } else { // Otherwise, improper indentation ends this item break; } + + if (!blankLine && !line.trim()) { // Check if current line is blank + blankLine = true; + } + + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); } if (!list.loose) { From c16536abae8700dfd964d824332cc7cd1080ff02 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sat, 27 Nov 2021 17:15:01 -0500 Subject: [PATCH 6/8] Update compiled files --- lib/marked.cjs | 68 +++++++++++++++++++---------------------------- lib/marked.esm.js | 64 +++++++++++++++++--------------------------- lib/marked.umd.js | 68 +++++++++++++++++++---------------------------- 3 files changed, 79 insertions(+), 121 deletions(-) diff --git a/lib/marked.cjs b/lib/marked.cjs index 7b3b49e544..374fc71980 100644 --- a/lib/marked.cjs +++ b/lib/marked.cjs @@ -533,7 +533,7 @@ var Tokenizer = /*#__PURE__*/function () { var cap = this.rules.block.list.exec(src); if (cap) { - var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, itemContents; + var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, nextLine, rawLine, itemContents; var bull = cap[1].trim(); var isordered = bull.length > 1; var list = { @@ -551,23 +551,22 @@ var Tokenizer = /*#__PURE__*/function () { } // Get next list item - var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*)?(?:\\n|$))"); // Get each top-level item + var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*)?(?:\\n|$))"); // Check if current bullet point can start a new List Item while (src) { - if (this.rules.block.hr.test(src)) { - // End list if we encounter an HR (possibly move into itemRegex?) + if (!(cap = itemRegex.exec(src))) { break; } - if (!(cap = itemRegex.exec(src))) { + if (this.rules.block.hr.test(src)) { + // End list if bullet was actually HR (possibly move into itemRegex?) break; - } //lines = cap[2].split('\n'); - + } raw = cap[0]; - line = cap[2].split('\n', 1)[0]; src = src.substring(raw.length); - var nextLine = src.split('\n', 1)[0]; + line = cap[2].split('\n', 1)[0]; + nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { indent = 2; @@ -584,58 +583,45 @@ var Tokenizer = /*#__PURE__*/function () { blankLine = false; if (!line && /^ *$/.test(nextLine)) { - // items begin with at most one blank line + // Items begin with at most one blank line raw += nextLine + '\n'; src = src.substring(nextLine.length + 1); list.loose = true; } - var nextBulletRegex = new RegExp("^ {0," + Math.min(3, indent - 1) + "}(?:[*+-]|\\d{1,9}[.)])"); - var rawLine = void 0; + var nextBulletRegex = new RegExp("^ {0," + Math.min(3, indent - 1) + "}(?:[*+-]|\\d{1,9}[.)])"); // Check if following lines should be included in List Item while (src && !list.loose) { rawLine = src.split('\n', 1)[0]; - line = rawLine; + line = rawLine; // Re-align to follow commonmark nesting rules if (this.options.pedantic) { - // Re-align to follow commonmark nesting rules line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); } // End list item if found start of new bullet if (nextBulletRegex.test(line)) { break; - } // Until we encounter a blank line, item contents do not need indentation - - - if (!blankLine) { - if (!line.trim()) { - // Check if current line is empty - blankLine = true; - } // Dedent if possible - - - if (line.search(/[^ ]/) >= indent) { - itemContents += '\n' + line.slice(indent); - } else { - itemContents += '\n' + line; - } - - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; - } // Dedent this line - + } if (line.search(/[^ ]/) >= indent || !line.trim()) { + // Dedent if possible itemContents += '\n' + line.slice(indent); - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; + } else if (!blankLine) { + // Until blank line, item doesn't need indentation + itemContents += '\n' + line; } else { - // Line was not properly indented; end of this item + // Otherwise, improper indentation ends this item break; } + + if (!blankLine && !line.trim()) { + // Check if current line is blank + blankLine = true; + } + + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); } if (!list.loose) { @@ -665,7 +651,7 @@ var Tokenizer = /*#__PURE__*/function () { loose: false, text: itemContents }); - list.raw += raw; //src = src.slice(raw.length); + list.raw += raw; } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -681,7 +667,7 @@ var Tokenizer = /*#__PURE__*/function () { if (!list.loose && list.items[i].tokens.some(function (t) { return t.type === 'space'; })) { - list.loose = true; //list.items[i].loose = true; + list.loose = true; // list.items[i].loose = true; } } diff --git a/lib/marked.esm.js b/lib/marked.esm.js index f961bc8e2d..d9ff3cdccd 100644 --- a/lib/marked.esm.js +++ b/lib/marked.esm.js @@ -452,7 +452,7 @@ class Tokenizer { let cap = this.rules.block.list.exec(src); if (cap) { let raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, - line, itemContents; + line, nextLine, rawLine, itemContents; let bull = cap[1].trim(); const isordered = bull.length > 1; @@ -475,21 +475,21 @@ class Tokenizer { // Get next list item const itemRegex = new RegExp(`^( {0,3}${bull})((?: [^\\n]*)?(?:\\n|$))`); - // Get each top-level item + // Check if current bullet point can start a new List Item while (src) { - if (this.rules.block.hr.test(src)) { // End list if we encounter an HR (possibly move into itemRegex?) + if (!(cap = itemRegex.exec(src))) { break; } - if (!(cap = itemRegex.exec(src))) { + if (this.rules.block.hr.test(src)) { // End list if bullet was actually HR (possibly move into itemRegex?) break; } - //lines = cap[2].split('\n'); raw = cap[0]; - line = cap[2].split('\n',1)[0]; src = src.substring(raw.length); - let nextLine = src.split('\n',1)[0]; + + line = cap[2].split('\n', 1)[0]; + nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { indent = 2; @@ -503,8 +503,7 @@ class Tokenizer { blankLine = false; - - if (!line && /^ *$/.test(nextLine)) { // items begin with at most one blank line + if (!line && /^ *$/.test(nextLine)) { // Items begin with at most one blank line raw += nextLine + '\n'; src = src.substring(nextLine.length + 1); list.loose = true; @@ -512,13 +511,13 @@ class Tokenizer { const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])`); - let rawLine; - - while(src && !list.loose) { - rawLine = src.split('\n',1)[0]; + // Check if following lines should be included in List Item + while (src && !list.loose) { + rawLine = src.split('\n', 1)[0]; line = rawLine; - if (this.options.pedantic) { // Re-align to follow commonmark nesting rules + // Re-align to follow commonmark nesting rules + if (this.options.pedantic) { line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); } @@ -527,32 +526,20 @@ class Tokenizer { break; } - // Until we encounter a blank line, item contents do not need indentation - if (!blankLine) { - if (!line.trim()) { // Check if current line is empty - blankLine = true; - } - - // Dedent if possible - if (line.search(/[^ ]/) >= indent) { - itemContents += '\n' + line.slice(indent); - } else { - itemContents += '\n' + line; - } - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; - } - - // Dedent this line - if (line.search(/[^ ]/) >= indent || !line.trim()) { + if (line.search(/[^ ]/) >= indent || !line.trim()) { // Dedent if possible itemContents += '\n' + line.slice(indent); - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; - } else { // Line was not properly indented; end of this item + } else if (!blankLine) { // Until blank line, item doesn't need indentation + itemContents += '\n' + line; + } else { // Otherwise, improper indentation ends this item break; } + + if (!blankLine && !line.trim()) { // Check if current line is blank + blankLine = true; + } + + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); } if (!list.loose) { @@ -583,7 +570,6 @@ class Tokenizer { }); list.raw += raw; - //src = src.slice(raw.length); } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -599,7 +585,7 @@ class Tokenizer { list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); if (!list.loose && list.items[i].tokens.some(t => t.type === 'space')) { list.loose = true; - //list.items[i].loose = true; + // list.items[i].loose = true; } } diff --git a/lib/marked.umd.js b/lib/marked.umd.js index d571e559e1..37dadbb7a8 100644 --- a/lib/marked.umd.js +++ b/lib/marked.umd.js @@ -535,7 +535,7 @@ var cap = this.rules.block.list.exec(src); if (cap) { - var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, itemContents; + var raw, istask, ischecked, indent, i, blankLine, endsWithBlankLine, line, nextLine, rawLine, itemContents; var bull = cap[1].trim(); var isordered = bull.length > 1; var list = { @@ -553,23 +553,22 @@ } // Get next list item - var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*)?(?:\\n|$))"); // Get each top-level item + var itemRegex = new RegExp("^( {0,3}" + bull + ")((?: [^\\n]*)?(?:\\n|$))"); // Check if current bullet point can start a new List Item while (src) { - if (this.rules.block.hr.test(src)) { - // End list if we encounter an HR (possibly move into itemRegex?) + if (!(cap = itemRegex.exec(src))) { break; } - if (!(cap = itemRegex.exec(src))) { + if (this.rules.block.hr.test(src)) { + // End list if bullet was actually HR (possibly move into itemRegex?) break; - } //lines = cap[2].split('\n'); - + } raw = cap[0]; - line = cap[2].split('\n', 1)[0]; src = src.substring(raw.length); - var nextLine = src.split('\n', 1)[0]; + line = cap[2].split('\n', 1)[0]; + nextLine = src.split('\n', 1)[0]; if (this.options.pedantic) { indent = 2; @@ -586,58 +585,45 @@ blankLine = false; if (!line && /^ *$/.test(nextLine)) { - // items begin with at most one blank line + // Items begin with at most one blank line raw += nextLine + '\n'; src = src.substring(nextLine.length + 1); list.loose = true; } - var nextBulletRegex = new RegExp("^ {0," + Math.min(3, indent - 1) + "}(?:[*+-]|\\d{1,9}[.)])"); - var rawLine = void 0; + var nextBulletRegex = new RegExp("^ {0," + Math.min(3, indent - 1) + "}(?:[*+-]|\\d{1,9}[.)])"); // Check if following lines should be included in List Item while (src && !list.loose) { rawLine = src.split('\n', 1)[0]; - line = rawLine; + line = rawLine; // Re-align to follow commonmark nesting rules if (this.options.pedantic) { - // Re-align to follow commonmark nesting rules line = line.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); } // End list item if found start of new bullet if (nextBulletRegex.test(line)) { break; - } // Until we encounter a blank line, item contents do not need indentation - - - if (!blankLine) { - if (!line.trim()) { - // Check if current line is empty - blankLine = true; - } // Dedent if possible - - - if (line.search(/[^ ]/) >= indent) { - itemContents += '\n' + line.slice(indent); - } else { - itemContents += '\n' + line; - } - - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; - } // Dedent this line - + } if (line.search(/[^ ]/) >= indent || !line.trim()) { + // Dedent if possible itemContents += '\n' + line.slice(indent); - raw += rawLine + '\n'; - src = src.substring(rawLine.length + 1); - continue; + } else if (!blankLine) { + // Until blank line, item doesn't need indentation + itemContents += '\n' + line; } else { - // Line was not properly indented; end of this item + // Otherwise, improper indentation ends this item break; } + + if (!blankLine && !line.trim()) { + // Check if current line is blank + blankLine = true; + } + + raw += rawLine + '\n'; + src = src.substring(rawLine.length + 1); } if (!list.loose) { @@ -667,7 +653,7 @@ loose: false, text: itemContents }); - list.raw += raw; //src = src.slice(raw.length); + list.raw += raw; } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic @@ -683,7 +669,7 @@ if (!list.loose && list.items[i].tokens.some(function (t) { return t.type === 'space'; })) { - list.loose = true; //list.items[i].loose = true; + list.loose = true; // list.items[i].loose = true; } } From 5ad1977a6ead3d831e91d79a6d44a43cf6cc8784 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Sat, 27 Nov 2021 21:10:30 -0500 Subject: [PATCH 7/8] Uncomment list.items.loose --- src/Tokenizer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 98e79850e0..a957883f28 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -302,7 +302,7 @@ export class Tokenizer { list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); if (!list.loose && list.items[i].tokens.some(t => t.type === 'space')) { list.loose = true; - // list.items[i].loose = true; + list.items[i].loose = true; } } From eea62d07816e7710b2f7b47c8146742a5caa6497 Mon Sep 17 00:00:00 2001 From: Trevor Buckner Date: Wed, 1 Dec 2021 16:13:13 -0500 Subject: [PATCH 8/8] Update lib files --- lib/marked.cjs | 3 ++- lib/marked.esm.js | 2 +- lib/marked.umd.js | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/marked.cjs b/lib/marked.cjs index ac02a4642b..97cbbe4b9f 100644 --- a/lib/marked.cjs +++ b/lib/marked.cjs @@ -667,7 +667,8 @@ var Tokenizer = /*#__PURE__*/function () { if (!list.loose && list.items[i].tokens.some(function (t) { return t.type === 'space'; })) { - list.loose = true; // list.items[i].loose = true; + list.loose = true; + list.items[i].loose = true; } } diff --git a/lib/marked.esm.js b/lib/marked.esm.js index e54af7e92c..c8fac1cf4e 100644 --- a/lib/marked.esm.js +++ b/lib/marked.esm.js @@ -585,7 +585,7 @@ class Tokenizer { list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); if (!list.loose && list.items[i].tokens.some(t => t.type === 'space')) { list.loose = true; - // list.items[i].loose = true; + list.items[i].loose = true; } } diff --git a/lib/marked.umd.js b/lib/marked.umd.js index 2a4f9961e4..25e84d3d21 100644 --- a/lib/marked.umd.js +++ b/lib/marked.umd.js @@ -669,7 +669,8 @@ if (!list.loose && list.items[i].tokens.some(function (t) { return t.type === 'space'; })) { - list.loose = true; // list.items[i].loose = true; + list.loose = true; + list.items[i].loose = true; } }