Fix #180 & #211: threat each newline as separate word in diffWordsWithSpace

Author: Mingun

src/diff/word.js

@@ -32,7 +32,8 @@ wordDiff.equals = function(left, right) {
   return left === right || (this.options.ignoreWhitespace && !reWhitespace.test(left) && !reWhitespace.test(right));
 };
 wordDiff.tokenize = function(value) {
-  let tokens = value.split(/(\s+|[()[\]{}'"]|\b)/);
+  // All whitespace symbols except newline group into one token, each newline - in separate token
+  let tokens = value.split(/([^\S\r\n]+|[()[\]{}'"\r\n]|\b)/);
 
   // Join the boundary splits that we do not consider to be boundaries. This is primarily the extended Latin character set.
   for (let i = 0; i < tokens.length - 1; i++) {

test/diff/word.js

@@ -209,6 +209,15 @@ describe('WordDiff', function() {
       expect(convertChangesToXML(diffResult)).to.equal('&quot;<ins>word</ins>&quot;');
     });
 
+    it('should threat newline as separate token (issues #180, #211)', function() {
+      // #180
+      const diffResult1 = diffWordsWithSpace('foo\nbar', 'foo\n\n\nbar');
+      expect(convertChangesToXML(diffResult1)).to.equal('foo\n<ins>\n\n</ins>bar');
+      // #211
+      const diffResult2 = diffWordsWithSpace('A\n\nB\n', 'A\nB\n');
+      expect(convertChangesToXML(diffResult2)).to.equal('A\n<del>\n</del>B\n');
+    });
+
     it('should perform async operations', function(done) {
       diffWordsWithSpace('New Value  ', 'New  ValueMoreData ', function(err, diffResult) {
         expect(convertChangesToXML(diffResult)).to.equal('New<ins>  ValueMoreData</ins> <del>Value  </del>');