Tests: Automatically normalize line ends (#2934)

Author: RunDevelopment

.gitattributes

@@ -1,4 +1 @@
 * text=auto
-
-# Test files should not have their line endings modified by git
-/tests/languages/**/*.test binary

components/prism-false.js

@@ -13,7 +13,7 @@
 			greedy: true
 		},
 		'character-code': {
-			pattern: /'[\s\S]/,
+			pattern: /'(?:[^\r]|\r\n?)/,
 			alias: 'number'
 		},
 		'assembler-code': {

components/prism-false.min.js

@@ -66,7 +66,6 @@ module.exports = {
 
 		function updateFile() {
 			// change the file
-			const lineEnd = (/\r\n/.test(testCase.code) || !/\n/.test(testCase.code)) ? '\r\n' : '\n';
 			const separator = '\n\n----------------------------------------------------\n\n';
 			const pretty = TokenStreamTransformer.prettyprint(tokenStream, '\t');
 
@@ -75,7 +74,9 @@ module.exports = {
 				content += separator + testCase.comment.trim();
 			}
 			content += '\n';
-			content = content.replace(/\r?\n/g, lineEnd);
+
+			// convert line ends to the line ends of the file
+			content = content.replace(/\r\n?|\n/g, testCase.lineEndOnDisk);
 
 			fs.writeFileSync(filePath, content, 'utf-8');
 		}
@@ -202,14 +203,19 @@ module.exports = {
 	 * @returns {ParsedTestCase}
 	 *
 	 * @typedef ParsedTestCase
+	 * @property {string} lineEndOnDisk The EOL format used by the parsed file.
 	 * @property {string} code
 	 * @property {string} expectedJson
 	 * @property {number} expectedLineOffset
 	 * @property {Array | null} expectedTokenStream
 	 * @property {string} comment
 	 */
 	parseTestCaseFile(filePath) {
-		const testCaseSource = fs.readFileSync(filePath, 'utf8');
+		let testCaseSource = fs.readFileSync(filePath, 'utf8');
+		const lineEndOnDisk = (/\r\n?|\n/.exec(testCaseSource) || ['\n'])[0];
+		// normalize line ends to \r\n
+		testCaseSource = testCaseSource.replace(/\r\n?|\n/g, '\r\n');
+
 		const testCaseParts = testCaseSource.split(/^-{10,}[ \t]*$/m);
 
 		if (testCaseParts.length > 3) {
@@ -221,9 +227,10 @@ module.exports = {
 		const comment = (testCaseParts[2] || '').trimStart();
 
 		const testCase = {
+			lineEndOnDisk,
 			code,
 			expectedJson: expected,
-			expectedLineOffset: code.split(/\r\n?|\n/g).length,
+			expectedLineOffset: code.split(/\r\n/g).length,
 			expectedTokenStream: expected ? JSON.parse(expected) : null,
 			comment
 		};

tests/helper/test-case.js

@@ -7,7 +7,7 @@
 ----------------------------------------------------
 
 [
-	["comment", "{-\n\tThis is a\n\tmultiline comment\n-}"],
+	["comment", "{-\r\n\tThis is a\r\n\tmultiline comment\r\n-}"],
 	["comment", "-- This is a singleline comment"]
 ]
 

tests/languages/agda/comment_feature.test

@@ -29,11 +29,11 @@ merge xs@(x ∷ xs₁) ys@(y ∷ ys₁) =
 	["operator", "→"],
 	" List A ",
 	["operator", "→"],
-	" List A\nmerge xs [] ",
+	" List A\r\nmerge xs [] ",
 	["operator", "="],
-	" xs\nmerge [] ys ",
+	" xs\r\nmerge [] ys ",
 	["operator", "="],
-	" ys\nmerge xs",
+	" ys\r\nmerge xs",
 	["punctuation", "@"],
 	["punctuation", "("],
 	"x ∷ xs₁",
@@ -44,7 +44,8 @@ merge xs@(x ∷ xs₁) ys@(y ∷ ys₁) =
 	"y ∷ ys₁",
 	["punctuation", ")"],
 	["operator", "="],
-	"\n\tif x < y then x ∷ merge xs₁ ys\n\t\t\t\t\t else y ∷ merge xs ys₁"
+
+	"\r\n\tif x < y then x ∷ merge xs₁ ys\r\n\t\t\t\t\t else y ∷ merge xs ys₁"
 ]
 
 ----------------------------------------------------

tests/languages/agda/function_feature.test

@@ -11,12 +11,14 @@ open Eq.≡-Reasoning using (begin_; _≡⟨⟩_; _∎) renaming (begin_ to star
 	["punctuation", "."],
 	"test ",
 	["keyword", "where"],
+
 	["keyword", "import"],
 	" Relation",
 	["punctuation", "."],
 	"Binary",
 	["punctuation", "."],
-	"PropositionalEquality as Eq\n",
+	"PropositionalEquality as Eq\r\n",
+
 	["keyword", "open"],
 	" Eq ",
 	["keyword", "hiding"],
@@ -25,6 +27,7 @@ open Eq.≡-Reasoning using (begin_; _≡⟨⟩_; _∎) renaming (begin_ to star
 	["punctuation", ";"],
 	" refl",
 	["punctuation", ")"],
+
 	["keyword", "open"],
 	" Eq",
 	["punctuation", "."],

tests/languages/agda/module_feature.test

@@ -18,15 +18,20 @@ open Fin
 	["operator", ":"],
 	["keyword", "Set"],
 	["keyword", "where"],
+
 	["keyword", "constructor"],
-	" _[_]\n\t",
+	" _[_]\r\n\t",
+
 	["keyword", "field"],
+
 	["function", "⟦_⟧   "],
 	["operator", ":"],
-	" Nat\n\t\t",
+	" Nat\r\n\t\t",
+
 	["function", "proof "],
 	["operator", ":"],
-	" suc ⟦_⟧ ≤ n\n",
+	" suc ⟦_⟧ ≤ n\r\n",
+
 	["keyword", "open"],
 	" Fin"
 ]

tests/languages/agda/record_feature.test

@@ -9,7 +9,7 @@
 [
 	["comment", "// comment"],
 	["comment", "/**/"],
-	["comment", "/*\n comment\n*/"]
+	["comment", "/*\r\n comment\r\n*/"]
 ]
 
 ----------------------------------------------------

tests/languages/al/comment_feature.test

@@ -21,12 +21,14 @@ from LexerAdaptor import LexerAdaptor
 		["content", " superClass = LexerAdaptor; "],
 		["punctuation", "}"]
 	]],
+
 	["annotation", "@lexer::header"],
 	["action", [
 		["punctuation", "{"],
-		["content", "\n  import { Token } from 'antlr4ts/Token';\n  import { CommonToken } from 'antlr4ts/CommonToken';\n  import { Python3Parser } from './Python3Parser';\n"],
+		["content", "\r\n  import { Token } from 'antlr4ts/Token';\r\n  import { CommonToken } from 'antlr4ts/CommonToken';\r\n  import { Python3Parser } from './Python3Parser';\r\n"],
 		["punctuation", "}"]
 	]],
+
 	["definition", "END"],
 	["punctuation", ":"],
 	["string", "'end'"],
@@ -36,10 +38,11 @@ from LexerAdaptor import LexerAdaptor
 		["punctuation", "}"]
 	]],
 	["punctuation", ";"],
+
 	["annotation", "@header"],
 	["action", [
 		["punctuation", "{"],
-		["content", "\nfrom LexerAdaptor import LexerAdaptor\n"],
+		["content", "\r\nfrom LexerAdaptor import LexerAdaptor\r\n"],
 		["punctuation", "}"]
 	]]
 ]