From 53c79ee9196bfd06ad15aad8904a7a4d61e0f400 Mon Sep 17 00:00:00 2001
From: Tony Brix <tony@brix.ninja>
Date: Tue, 26 Jan 2021 08:20:13 -0600
Subject: [PATCH] fix: leave whitespace only lines alone (#1889)

* fix: leave whitespace only lines alone

* test: add whitepace_lines test

* fix: render code with trailing new line
---
 src/Lexer.js                                 |  4 +++-
 src/Renderer.js                              |  2 ++
 src/Tokenizer.js                             |  2 +-
 src/rules.js                                 |  6 +++---
 test/specs/new/code_compensation_indent.html |  3 ++-
 test/specs/new/code_consistent_newline.html  |  6 ++++--
 test/specs/new/whiltespace_lines.html        | 12 ++++++++++++
 test/specs/new/whiltespace_lines.md          | 18 ++++++++++++++++++
 test/unit/marked-spec.js                     | 18 ++++++++++++------
 9 files changed, 57 insertions(+), 14 deletions(-)
 create mode 100644 test/specs/new/whiltespace_lines.html
 create mode 100644 test/specs/new/whiltespace_lines.md

diff --git a/src/Lexer.js b/src/Lexer.js
index 493ce9e6ea..bb04b43101 100644
--- a/src/Lexer.js
+++ b/src/Lexer.js
@@ -120,7 +120,9 @@ module.exports = class Lexer {
    * Lexing
    */
   blockTokens(src, tokens = [], top = true) {
-    src = src.replace(/^ +$/gm, '');
+    if (this.options.pedantic) {
+      src = src.replace(/^ +$/gm, '');
+    }
     let token, i, l, lastToken;
 
     while (src) {
diff --git a/src/Renderer.js b/src/Renderer.js
index ace12381bb..1fa9714602 100644
--- a/src/Renderer.js
+++ b/src/Renderer.js
@@ -22,6 +22,8 @@ module.exports = class Renderer {
       }
     }
 
+    code = code.replace(/\n$/, '') + '\n';
+
     if (!lang) {
       return '<pre><code>'
         + (escaped ? code : escape(code, true))
diff --git a/src/Tokenizer.js b/src/Tokenizer.js
index cb2a275a22..f01c1ca8d3 100644
--- a/src/Tokenizer.js
+++ b/src/Tokenizer.js
@@ -91,7 +91,7 @@ module.exports = class Tokenizer {
         };
       }
 
-      const text = cap[0].replace(/^ {4}/gm, '');
+      const text = cap[0].replace(/^ {1,4}/gm, '');
       return {
         type: 'code',
         raw: cap[0],
diff --git a/src/rules.js b/src/rules.js
index 69975e189a..b7f245e714 100644
--- a/src/rules.js
+++ b/src/rules.js
@@ -8,8 +8,8 @@ const {
  * Block-Level Grammar
  */
 const block = {
-  newline: /^\n+/,
-  code: /^( {4}[^\n]+\n*)+/,
+  newline: /^(?: *(?:\n|$))+/,
+  code: /^( {4}[^\n]+(?:\n(?: *(?:\n|$))*)?)+/,
   fences: /^ {0,3}(`{3,}(?=[^`\n]*\n)|~{3,})([^\n]*)\n(?:|([\s\S]*?)\n)(?: {0,3}\1[~`]* *(?:\n+|$)|$)/,
   hr: /^ {0,3}((?:- *){3,}|(?:_ *){3,}|(?:\* *){3,})(?:\n+|$)/,
   heading: /^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/,
@@ -31,7 +31,7 @@ const block = {
   lheading: /^([^\n]+)\n {0,3}(=+|-+) *(?:\n+|$)/,
   // regex template, placeholders will be replaced according to different paragraph
   // interruption rules of commonmark and the original markdown spec:
-  _paragraph: /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html)[^\n]+)*)/,
+  _paragraph: /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html| +\n)[^\n]+)*)/,
   text: /^[^\n]+/
 };
 
diff --git a/test/specs/new/code_compensation_indent.html b/test/specs/new/code_compensation_indent.html
index b6b7d227b4..4c2ebefb8a 100644
--- a/test/specs/new/code_compensation_indent.html
+++ b/test/specs/new/code_compensation_indent.html
@@ -2,6 +2,7 @@
 <ol>
 <li><p>This is a list element.</p>
 <pre><code>const x = 5;
-const y = x + 5;</code></pre>
+const y = x + 5;
+</code></pre>
 </li>
 </ol>
diff --git a/test/specs/new/code_consistent_newline.html b/test/specs/new/code_consistent_newline.html
index f1ebc6fc8d..41a15c4ee8 100644
--- a/test/specs/new/code_consistent_newline.html
+++ b/test/specs/new/code_consistent_newline.html
@@ -1,3 +1,5 @@
-<pre><code class="language-js">const value = 42;</code></pre>
-<pre><code>const value = 42;</code></pre>
+<pre><code class="language-js">const value = 42;
+</code></pre>
+<pre><code>const value = 42;
+</code></pre>
 <p>Code blocks contain trailing new line.</p>
diff --git a/test/specs/new/whiltespace_lines.html b/test/specs/new/whiltespace_lines.html
new file mode 100644
index 0000000000..97931fe26a
--- /dev/null
+++ b/test/specs/new/whiltespace_lines.html
@@ -0,0 +1,12 @@
+<p>paragraph</p>
+<p>test</p>
+<pre><code>a
+  
+b
+
+c
+</code></pre>
+<pre><code>a
+  
+b
+</code></pre>
diff --git a/test/specs/new/whiltespace_lines.md b/test/specs/new/whiltespace_lines.md
new file mode 100644
index 0000000000..ff645b3842
--- /dev/null
+++ b/test/specs/new/whiltespace_lines.md
@@ -0,0 +1,18 @@
+---
+renderExact: true
+---
+paragraph
+  
+test
+
+    a
+      
+    b
+  
+    c
+
+```
+a
+  
+b
+```
diff --git a/test/unit/marked-spec.js b/test/unit/marked-spec.js
index fcee51a3eb..7d81117288 100644
--- a/test/unit/marked-spec.js
+++ b/test/unit/marked-spec.js
@@ -335,12 +335,15 @@ text 1
         fail(err);
       }
 
-      expect(html).toBe(`<pre><code class="language-lang1">async text 1</code></pre>
+      expect(html).toBe(`<pre><code class="language-lang1">async text 1
+</code></pre>
 <blockquote>
-<pre><code class="language-lang2">async text 2</code></pre>
+<pre><code class="language-lang2">async text 2
+</code></pre>
 </blockquote>
 <ul>
-<li><pre><code class="language-lang3">async text 3</code></pre>
+<li><pre><code class="language-lang3">async text 3
+</code></pre>
 </li>
 </ul>
 `);
@@ -378,12 +381,15 @@ text 1
         fail(err);
       }
 
-      expect(html).toBe(`<pre><code class="language-lang1">async text 1</code></pre>
+      expect(html).toBe(`<pre><code class="language-lang1">async text 1
+</code></pre>
 <blockquote>
-<pre><code class="language-lang2">async text 2</code></pre>
+<pre><code class="language-lang2">async text 2
+</code></pre>
 </blockquote>
 <ul>
-<li><pre><code class="language-lang3">async text 3</code></pre>
+<li><pre><code class="language-lang3">async text 3
+</code></pre>
 </li>
 </ul>
 `);