Tests: Automatically normalize line ends (#2934)

Author: RunDevelopment

.gitattributes

@@ -1,4 +1 @@
 * text=auto
-
-# Test files should not have their line endings modified by git
-/tests/languages/**/*.test binary

components/prism-false.js

@@ -13,7 +13,7 @@
 			greedy: true
 		},
 		'character-code': {
-			pattern: /'[\s\S]/,
+			pattern: /'(?:[^\r]|\r\n?)/,
 			alias: 'number'
 		},
 		'assembler-code': {

components/prism-false.min.js

@@ -66,7 +66,6 @@ module.exports = {
 
 		function updateFile() {
 			// change the file
-			const lineEnd = (/\r\n/.test(testCase.code) || !/\n/.test(testCase.code)) ? '\r\n' : '\n';
 			const separator = '\n\n----------------------------------------------------\n\n';
 			const pretty = TokenStreamTransformer.prettyprint(tokenStream, '\t');
 
@@ -75,7 +74,9 @@ module.exports = {
 				content += separator + testCase.comment.trim();
 			}
 			content += '\n';
-			content = content.replace(/\r?\n/g, lineEnd);
+
+			// convert line ends to the line ends of the file
+			content = content.replace(/\r\n?|\n/g, testCase.lineEndOnDisk);
 
 			fs.writeFileSync(filePath, content, 'utf-8');
 		}
@@ -202,14 +203,19 @@ module.exports = {
 	 * @returns {ParsedTestCase}
 	 *
 	 * @typedef ParsedTestCase
+	 * @property {string} lineEndOnDisk The EOL format used by the parsed file.
 	 * @property {string} code
 	 * @property {string} expectedJson
 	 * @property {number} expectedLineOffset
 	 * @property {Array | null} expectedTokenStream
 	 * @property {string} comment
 	 */
 	parseTestCaseFile(filePath) {
-		const testCaseSource = fs.readFileSync(filePath, 'utf8');
+		let testCaseSource = fs.readFileSync(filePath, 'utf8');
+		const lineEndOnDisk = (/\r\n?|\n/.exec(testCaseSource) || ['\n'])[0];
+		// normalize line ends to \r\n
+		testCaseSource = testCaseSource.replace(/\r\n?|\n/g, '\r\n');
+
 		const testCaseParts = testCaseSource.split(/^-{10,}[ \t]*$/m);
 
 		if (testCaseParts.length > 3) {
@@ -221,9 +227,10 @@ module.exports = {
 		const comment = (testCaseParts[2] || '').trimStart();
 
 		const testCase = {
+			lineEndOnDisk,
 			code,
 			expectedJson: expected,
-			expectedLineOffset: code.split(/\r\n?|\n/g).length,
+			expectedLineOffset: code.split(/\r\n/g).length,
 			expectedTokenStream: expected ? JSON.parse(expected) : null,
 			comment
 		};

tests/helper/test-case.js

@@ -7,7 +7,7 @@
 ----------------------------------------------------
 
 [
-	["comment", "{-\n\tThis is a\n\tmultiline comment\n-}"],
+	["comment", "{-\r\n\tThis is a\r\n\tmultiline comment\r\n-}"],
 	["comment", "-- This is a singleline comment"]
 ]
 

tests/languages/agda/comment_feature.test

@@ -29,11 +29,11 @@ merge xs@(x ∷ xs₁) ys@(y ∷ ys₁) =
 	["operator", "→"],
 	" List A ",
 	["operator", "→"],
-	" List A\nmerge xs [] ",
+	" List A\r\nmerge xs [] ",
 	["operator", "="],
-	" xs\nmerge [] ys ",
+	" xs\r\nmerge [] ys ",
 	["operator", "="],
-	" ys\nmerge xs",
+	" ys\r\nmerge xs",
 	["punctuation", "@"],
 	["punctuation", "("],
 	"x ∷ xs₁",
@@ -44,7 +44,8 @@ merge xs@(x ∷ xs₁) ys@(y ∷ ys₁) =
 	"y ∷ ys₁",
 	["punctuation", ")"],
 	["operator", "="],
-	"\n\tif x < y then x ∷ merge xs₁ ys\n\t\t\t\t\t else y ∷ merge xs ys₁"
+
+	"\r\n\tif x < y then x ∷ merge xs₁ ys\r\n\t\t\t\t\t else y ∷ merge xs ys₁"
 ]
 
 ----------------------------------------------------

tests/languages/agda/function_feature.test

@@ -11,12 +11,14 @@ open Eq.≡-Reasoning using (begin_; _≡⟨⟩_; _∎) renaming (begin_ to star
 	["punctuation", "."],
 	"test ",
 	["keyword", "where"],
+
 	["keyword", "import"],
 	" Relation",
 	["punctuation", "."],
 	"Binary",
 	["punctuation", "."],
-	"PropositionalEquality as Eq\n",
+	"PropositionalEquality as Eq\r\n",
+
 	["keyword", "open"],
 	" Eq ",
 	["keyword", "hiding"],
@@ -25,6 +27,7 @@ open Eq.≡-Reasoning using (begin_; _≡⟨⟩_; _∎) renaming (begin_ to star
 	["punctuation", ";"],
 	" refl",
 	["punctuation", ")"],
+
 	["keyword", "open"],
 	" Eq",
 	["punctuation", "."],

tests/languages/agda/module_feature.test

@@ -18,15 +18,20 @@ open Fin
 	["operator", ":"],
 	["keyword", "Set"],
 	["keyword", "where"],
+
 	["keyword", "constructor"],
-	" _[_]\n\t",
+	" _[_]\r\n\t",
+
 	["keyword", "field"],
+
 	["function", "⟦_⟧   "],
 	["operator", ":"],
-	" Nat\n\t\t",
+	" Nat\r\n\t\t",
+
 	["function", "proof "],
 	["operator", ":"],
-	" suc ⟦_⟧ ≤ n\n",
+	" suc ⟦_⟧ ≤ n\r\n",
+
 	["keyword", "open"],
 	" Fin"
 ]

tests/languages/agda/record_feature.test

@@ -9,7 +9,7 @@
 [
 	["comment", "// comment"],
 	["comment", "/**/"],
-	["comment", "/*\n comment\n*/"]
+	["comment", "/*\r\n comment\r\n*/"]
 ]
 
 ----------------------------------------------------

tests/languages/al/comment_feature.test

@@ -21,12 +21,14 @@ from LexerAdaptor import LexerAdaptor
 		["content", " superClass = LexerAdaptor; "],
 		["punctuation", "}"]
 	]],
+
 	["annotation", "@lexer::header"],
 	["action", [
 		["punctuation", "{"],
-		["content", "\n  import { Token } from 'antlr4ts/Token';\n  import { CommonToken } from 'antlr4ts/CommonToken';\n  import { Python3Parser } from './Python3Parser';\n"],
+		["content", "\r\n  import { Token } from 'antlr4ts/Token';\r\n  import { CommonToken } from 'antlr4ts/CommonToken';\r\n  import { Python3Parser } from './Python3Parser';\r\n"],
 		["punctuation", "}"]
 	]],
+
 	["definition", "END"],
 	["punctuation", ":"],
 	["string", "'end'"],
@@ -36,10 +38,11 @@ from LexerAdaptor import LexerAdaptor
 		["punctuation", "}"]
 	]],
 	["punctuation", ";"],
+
 	["annotation", "@header"],
 	["action", [
 		["punctuation", "{"],
-		["content", "\nfrom LexerAdaptor import LexerAdaptor\n"],
+		["content", "\r\nfrom LexerAdaptor import LexerAdaptor\r\n"],
 		["punctuation", "}"]
 	]]
 ]

tests/languages/antlr4/action_feature.test

@@ -7,7 +7,7 @@
 
 [
 	["comment", "// comment"],
-	["comment", "/*\n comment\n*/"]
+	["comment", "/*\r\n comment\r\n*/"]
 ]
 
 ----------------------------------------------------

tests/languages/antlr4/comment_feature.test

@@ -29,22 +29,25 @@
 		]],
 		["punctuation", "]"]
 	]],
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
 			"*"
 		]],
 		["punctuation", "]"]
 	]],
-	"Entry A\n  ",
+	"Entry A\r\n  ",
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
 			"*"
 		]],
 		["punctuation", "]"]
 	]],
-	"Entry B\n",
+	"Entry B\r\n",
+
 	["tag", [
 		["tag", [
 			["punctuation", "[/"],
@@ -64,22 +67,25 @@
 		]],
 		["punctuation", "]"]
 	]],
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
 			"*"
 		]],
 		["punctuation", "]"]
 	]],
-	"Entry 1\n  ",
+	"Entry 1\r\n  ",
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
 			"*"
 		]],
 		["punctuation", "]"]
 	]],
-	"Entry 2\n",
+	"Entry 2\r\n",
+
 	["tag", [
 		["tag", [
 			["punctuation", "[/"],
@@ -110,7 +116,8 @@
 		]],
 		["punctuation", "]"]
 	]],
-	" or\n",
+	" or\r\n",
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
@@ -131,7 +138,8 @@
 		]],
 		["punctuation", "]"]
 	]],
-	" or\n",
+	" or\r\n",
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
@@ -151,6 +159,7 @@
 		]],
 		["punctuation", "]"]
 	]],
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
@@ -166,6 +175,7 @@
 		]],
 		["punctuation", "]"]
 	]],
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],
@@ -211,6 +221,7 @@
 		]],
 		["punctuation", "]"]
 	]],
+
 	["tag", [
 		["tag", [
 			["punctuation", "["],

tests/languages/bbcode/tag_feature.test

@@ -9,15 +9,11 @@ a = { foo: 5, bar: 6, "foo bar": 7 }
 
 [
 	["punctuation", "{"],
-	["property", "foo"],
-	["operator", ":"],
-	["number", "4"],
-	["property", "\"bar\""],
-	["operator", ":"],
-	["number", "5"],
+	["property", "foo"], ["operator", ":"], ["number", "4"],
+	["property", "\"bar\""], ["operator", ":"], ["number", "5"],
 	["punctuation", "}"],
 
-	"\n\na ",
+	"\r\n\r\na ",
 	["operator", "="],
 	["punctuation", "{"],
 	["property", "foo"],

tests/languages/brightscript/property_feature.test

@@ -8,7 +8,7 @@ string"
 [
 	["string", "\"\""],
 	["string", "\"Fo\\\"obar\""],
-	["string", "\"multi-line\nstring\""]
+	["string", "\"multi-line\r\nstring\""]
 ]
 
 ----------------------------------------------------

tests/languages/clojure/string_feature.test

@@ -10,29 +10,39 @@ string"
 
 [
 	["string", ["\"This is a string\""]],
-	["string", ["\"This is \nmulti\nline\nstring\""]],
+	["string", ["\"This is \r\nmulti\r\nline\r\nstring\""]],
 	["string", [
 		"\"",
 		["interpolation", [
-			["punctuation", "${"], ["variable", "VAR"], ["punctuation", "}"]]
-		],
+			["punctuation", "${"],
+			["variable", "VAR"],
+			["punctuation", "}"]
+		]],
 		"with",
 		["interpolation", [
-			["punctuation", "${"], ["variable", "BAR"], ["punctuation", "}"]]
-		], "\""]
-	],
+			["punctuation", "${"],
+			["variable", "BAR"],
+			["punctuation", "}"]
+		]],
+		"\""
+	]],
 	["string", [
 		"\"",
 		["interpolation", [
-			["punctuation", "${"], ["variable", "FOO"], ["punctuation", "}"]]
-		],
+			["punctuation", "${"],
+			["variable", "FOO"],
+			["punctuation", "}"]
+		]],
 		" with ",
 		["interpolation", [
-			["punctuation", "${"], ["variable", "BAR"], ["punctuation", "}"]]
-		], "\""]
-	]
+			["punctuation", "${"],
+			["variable", "BAR"],
+			["punctuation", "}"]
+		]],
+		"\""
+	]]
 ]
 
 ----------------------------------------------------
 
-Checks for strings.
+Checks for strings.

tests/languages/cmake/string_feature.test

@@ -69,7 +69,7 @@ greeting pic x(12) value is "Hello World".
 	]],
 	["punctuation", "."],
 
-	"\n\ngreeting ",
+	"\r\n\r\ngreeting ",
 	["keyword", "pic"],
 	["class-name", [
 		"x",
@@ -81,4 +81,4 @@ greeting pic x(12) value is "Hello World".
 	["keyword", "is"],
 	["string", "\"Hello World\""],
 	["punctuation", "."]
-]
+]