Handle unprocessable whitespace-related unicode characters

Author: TTOzzi

Sources/SwiftFormat/PrettyPrint/WhitespaceLinter.swift

@@ -312,13 +312,14 @@ public class WhitespaceLinter {
     formattedRun: ArraySlice<UTF8.CodeUnit>
   ) {
     guard userRun != formattedRun else { return }
-
+    let userString = String(decoding: userRun, as: UTF8.self)
+    let formattedString = String(decoding: formattedRun, as: UTF8.self)
     // This assumes tabs will always be forbidden for inter-token spacing (but not for leading
     // indentation).
-    if userRun.contains(utf8Tab) {
+    if userString.contains("\t") {
       diagnose(.spacingCharError, category: .spacingCharacter, utf8Offset: userIndex)
-    } else if formattedRun.count != userRun.count {
-      let delta = formattedRun.count - userRun.count
+    } else if formattedString.count != userString.count {
+      let delta = formattedString.count - userString.count
       diagnose(.spacingError(delta), category: .spacing, utf8Offset: userIndex)
     }
   }
@@ -339,20 +340,15 @@ public class WhitespaceLinter {
     startingAt offset: Int,
     in data: [UTF8.CodeUnit]
   ) -> ArraySlice<UTF8.CodeUnit> {
-    func isWhitespace(_ char: UTF8.CodeUnit) -> Bool {
-      switch char {
-      case UInt8(ascii: " "), UInt8(ascii: "\n"), UInt8(ascii: "\t"), UInt8(ascii: "\r"), /*VT*/ 0x0B, /*FF*/ 0x0C:
-        return true
-      default:
-        return false
-      }
-    }
-    guard
-      let whitespaceEnd = data[offset...].firstIndex(where: { !isWhitespace($0) })
-    else {
-      return data[offset..<data.endIndex]
+    // To ensure consistency with PrettyPrinter, the data is converted to a String
+    // before processing whitespace.
+    let substring = String(decoding: data[offset...], as: UTF8.self)
+    var stringIndex = substring.startIndex
+    while stringIndex < substring.endIndex, substring[stringIndex].isWhitespace {
+      substring.formIndex(after: &stringIndex)
     }
-    return data[offset..<whitespaceEnd]
+    let utf8Count = substring.utf8.distance(from: substring.startIndex, to: stringIndex)
+    return data[offset..<offset + utf8Count]
   }
 
   /// Returns the code unit at the given index, or nil if the index is the end of the data.

Tests/SwiftFormatTests/PrettyPrint/WhitespaceLintTests.swift

@@ -255,4 +255,27 @@ final class WhitespaceLintTests: WhitespaceTestCase {
       ]
     )
   }
+
+  func testUnexpectedUnicodeCharacters() {
+    assertWhitespaceLint(
+      input: """
+        // Hello World1️⃣\u{2028}
+        // Hello2️⃣\u{20}\u{2028}World
+        // Hello World3️⃣\u{2028}\u{2029}\u{2029}
+        // Hello World4️⃣\u{20}\u{20}\u{20}\u{2028}
+        """,
+      expected: """
+        // Hello World
+        // Hello World
+        // Hello World
+        // Hello World
+        """,
+      findings: [
+        FindingSpec("1️⃣", message: "remove trailing whitespace"),
+        FindingSpec("2️⃣", message: "remove 1 space"),
+        FindingSpec("3️⃣", message: "remove trailing whitespace"),
+        FindingSpec("4️⃣", message: "remove 4 spaces"),
+      ]
+    )
+  }
 }