Use emoji to mark the SourceEdits

This change is used to help us mark the changes in source file, which makes incremental parsing test easier.

Author: StevenWong12

Sources/_SwiftSyntaxTestSupport/SourceEditsTestUtils.swift

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import XCTest
+
+/// Get `ConcurrentEdits` in source whose edited zones are marked with markers
+/// Also extract the markers from source to get original source and edited source
+///
+/// `⏩️` is  *start marker*, `⏸️` is *separate marker*, `⏪️` is *end marker*
+/// Contents between `⏩️` and `⏸️` are source text that before modification, contents
+/// betwwen `⏸️` and `⏪️` are source text that after modification
+/// i.e. `⏩️foo⏸️bar⏪️`, the original source is `foo` and the edited source is `bar`
+public func getEditsAndSources(_ source: String) -> (edits: ConcurrentEdits, orignialSource: Substring, editedSource: Substring) {
+  var editedSource = source.prefix(0)
+  var originalSource = source.prefix(0)
+  var sequentialEdits: [SourceEdit] = []
+
+  var lastStartIndex = source.startIndex
+  while let startIndex = source[lastStartIndex...].firstIndex(where: { $0.isStartMarker }),
+    let separateIndex = source[startIndex...].firstIndex(where: { $0.isSeperateMarker }),
+    let endIndex = source[separateIndex...].firstIndex(where: { $0.isEndMarker })
+  {
+
+    editedSource += source[lastStartIndex..<startIndex]
+    let edit = SourceEdit(offset: editedSource.utf8.count, length: source.utf8.distance(from: source.index(after: startIndex), to: separateIndex), replacementLength: source.utf8.distance(from: source.index(after: separateIndex), to: endIndex))
+    editedSource += source[source.index(after: separateIndex)..<endIndex]
+
+    originalSource += source[lastStartIndex..<startIndex] + source[source.index(after: startIndex)..<separateIndex]
+
+    sequentialEdits.append(edit)
+
+    lastStartIndex = source.index(after: endIndex)
+  }
+
+  editedSource += source[lastStartIndex...]
+  originalSource += source[lastStartIndex...]
+
+  return (ConcurrentEdits(fromSequential: sequentialEdits), originalSource, editedSource)
+}
+
+/// Apply the given edits to `testString` and return the resulting string.
+/// `concurrent` specifies whether the edits should be interpreted as being
+/// applied sequentially or concurrently.
+public func applyEdits(
+  _ edits: [SourceEdit],
+  concurrent: Bool,
+  to testString: String,
+  replacementChar: Character = "?"
+) -> String {
+  guard let replacementAscii = replacementChar.asciiValue else {
+    fatalError("replacementChar must be an ASCII character")
+  }
+  var edits = edits
+  if concurrent {
+    XCTAssert(ConcurrentEdits._isValidConcurrentEditArray(edits))
+
+    // If the edits are concurrent, sorted and not overlapping (as guaranteed by
+    // the check above, we can apply them sequentially to the string in reverse
+    // order because later edits don't affect earlier edits.
+    edits = edits.reversed()
+  }
+  var bytes = Array(testString.utf8)
+  for edit in edits {
+    assert(edit.endOffset <= bytes.count)
+    bytes.removeSubrange(edit.offset..<edit.endOffset)
+    bytes.insert(contentsOf: [UInt8](repeating: replacementAscii, count: edit.replacementLength), at: edit.offset)
+  }
+  return String(bytes: bytes, encoding: .utf8)!
+}
+
+fileprivate extension Character {
+  var isStartMarker: Bool {
+    self == "⏩️"
+  }
+
+  var isSeperateMarker: Bool {
+    self == "⏸️"
+  }
+
+  var isEndMarker: Bool {
+    self == "⏪️"
+  }
+}

Tests/SwiftParserTest/IncrementalParsingTests.swift

@@ -13,34 +13,43 @@
 import XCTest
 import SwiftSyntax
 import SwiftParser
+import _SwiftSyntaxTestSupport
 
 public class IncrementalParsingTests: XCTestCase {
 
   public func testIncrementalInvalid() {
-    let original = "struct A { func f() {"
-    let step: (String, (Int, Int, String)) =
-      ("struct AA { func f() {", (8, 0, "A"))
-
-    var tree = Parser.parse(source: original)
-    let sourceEdit = SourceEdit(range: ByteSourceRange(offset: step.1.0, length: step.1.1), replacementLength: step.1.2.utf8.count)
-    let lookup = IncrementalParseTransition(previousTree: tree, edits: ConcurrentEdits(sourceEdit))
-    tree = Parser.parse(source: step.0, parseTransition: lookup)
-    XCTAssertEqual("\(tree)", step.0)
+    let source = "struct A⏩️⏸️A⏪️ { func f() {"
+    let (concurrentEdits, originalSubString, editedSubString) = getEditsAndSources(source)
+
+    let originalSource = String(originalSubString)
+    let editedSource = String(editedSubString)
+
+    var tree = Parser.parse(source: originalSource)
+
+    let lookup = IncrementalParseTransition(previousTree: tree, edits: concurrentEdits)
+    tree = Parser.parse(source: editedSource, parseTransition: lookup)
+    XCTAssertEqual("\(tree)", editedSource)
   }
 
   public func testReusedNode() throws {
     try XCTSkipIf(true, "Swift parser does not handle node reuse yet")
 
-    let original = "struct A {}\nstruct B {}\n"
-    let step: (String, (Int, Int, String)) =
-      ("struct AA {}\nstruct B {}\n", (8, 0, "A"))
+    let source =
+      """
+      struct A⏩️⏸️A⏪️ {}
+      struct B {}
+      """
+
+    let (concurrentEdits, originalSubString, editedSubString) = getEditsAndSources(source)
+
+    let originalSource = String(originalSubString)
+    let editedSource = String(editedSubString)
 
-    let origTree = Parser.parse(source: original)
-    let sourceEdit = SourceEdit(range: ByteSourceRange(offset: step.1.0, length: step.1.1), replacementLength: step.1.2.utf8.count)
+    let origTree = Parser.parse(source: originalSource)
     let reusedNodeCollector = IncrementalParseReusedNodeCollector()
-    let transition = IncrementalParseTransition(previousTree: origTree, edits: ConcurrentEdits(sourceEdit), reusedNodeDelegate: reusedNodeCollector)
-    let newTree = Parser.parse(source: step.0, parseTransition: transition)
-    XCTAssertEqual("\(newTree)", step.0)
+    let transition = IncrementalParseTransition(previousTree: origTree, edits: concurrentEdits, reusedNodeDelegate: reusedNodeCollector)
+    let newTree = Parser.parse(source: editedSource, parseTransition: transition)
+    XCTAssertEqual("\(newTree)", editedSource)
 
     let origStructB = origTree.statements[1]
     let newStructB = newTree.statements[1]

Tests/SwiftSyntaxTest/SequentialToConcurrentEditTranslationTests.swift

@@ -12,6 +12,7 @@
 
 import XCTest
 import SwiftSyntax
+import _SwiftSyntaxTestSupport
 
 let longString = """
   1234567890abcdefghijklmnopqrstuvwxyz\
@@ -26,36 +27,6 @@ let longString = """
   1234567890abcdefghijklmnopqrstuvwzyz
   """
 
-/// Apply the given edits to `testString` and return the resulting string.
-/// `concurrent` specifies whether the edits should be interpreted as being
-/// applied sequentially or concurrently.
-func applyEdits(
-  _ edits: [SourceEdit],
-  concurrent: Bool,
-  to testString: String = longString,
-  replacementChar: Character = "?"
-) -> String {
-  guard let replacementAscii = replacementChar.asciiValue else {
-    fatalError("replacementChar must be an ASCII character")
-  }
-  var edits = edits
-  if concurrent {
-    XCTAssert(ConcurrentEdits._isValidConcurrentEditArray(edits))
-
-    // If the edits are concurrent, sorted and not overlapping (as guaranteed by
-    // the check above, we can apply them sequentially to the string in reverse
-    // order because later edits don't affect earlier edits.
-    edits = edits.reversed()
-  }
-  var bytes = Array(testString.utf8)
-  for edit in edits {
-    assert(edit.endOffset <= bytes.count)
-    bytes.removeSubrange(edit.offset..<edit.endOffset)
-    bytes.insert(contentsOf: [UInt8](repeating: replacementAscii, count: edit.replacementLength), at: edit.offset)
-  }
-  return String(bytes: bytes, encoding: .utf8)!
-}
-
 /// Verifies that
 ///  1. translation of the `sequential` edits results in the
 ///     `expectedConcurrent` edits
@@ -364,7 +335,7 @@ final class TranslateSequentialToConcurrentEditsTests: XCTestCase {
       }
       print(edits)
       let normalizedEdits = ConcurrentEdits(fromSequential: edits)
-      if applyEdits(edits, concurrent: false) != applyEdits(normalizedEdits.edits, concurrent: true) {
+      if applyEdits(edits, concurrent: false, to: longString) != applyEdits(normalizedEdits.edits, concurrent: true, to: longString) {
         print("failed \(i)")
         fatalError()
       } else {

Tests/SwiftSyntaxTestSupportTest/SourceEditsTestUtilsTest.swift

@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import XCTest
+import SwiftSyntax
+import _SwiftSyntaxTestSupport
+
+public class SourceEditsUtilTest: XCTestCase {
+  public func testGetConcurrentEdits() {
+    let source =
+      """
+      ⏩️class⏸️struct⏪️ foo {
+        init() {
+          ⏩️⏸️let bar = 10⏪️
+        }
+
+        ⏩️func bar() {}⏸️⏪️
+      }
+      """
+
+    let (concurrentEdits, originalSource, _) = getEditsAndSources(source)
+
+    XCTAssertEqual(
+      concurrentEdits.edits,
+      [
+        SourceEdit(offset: 0, length: 5, replacementLength: 6),
+        SourceEdit(offset: 27, length: 0, replacementLength: 12),
+        SourceEdit(offset: 35, length: 13, replacementLength: 0),
+      ]
+    )
+
+    let expectedSource =
+      """
+      ?????? foo {
+        init() {
+          ????????????
+        }
+
+        
+      }
+      """
+
+    let sourceAppliedEdits = applyEdits(concurrentEdits.edits, concurrent: true, to: String(originalSource))
+
+    XCTAssertEqual(sourceAppliedEdits, expectedSource)
+  }
+
+  public func testReplaceMultiByteCharWithShorter() {
+    let source = "⏩️👨‍👩‍👧‍👦⏸️🎉⏪️"
+
+    let (concurrentEdits, originalSource, editedSource) = getEditsAndSources(source)
+
+    XCTAssertEqual(String(originalSource), "👨‍👩‍👧‍👦")
+    XCTAssertEqual(String(editedSource), "🎉")
+    XCTAssertEqual(
+      concurrentEdits.edits,
+      [
+        SourceEdit(offset: 0, length: 25, replacementLength: 4)
+      ]
+    )
+  }
+
+  public func testReplaceWithMultiByteChar() {
+    let source = "⏩️a⏸️👨‍👩‍👧‍👦⏪️"
+
+    let (concurrentEdits, originalSource, editedSource) = getEditsAndSources(source)
+
+    XCTAssertEqual(String(originalSource), "a")
+    XCTAssertEqual(String(editedSource), "👨‍👩‍👧‍👦")
+    XCTAssertEqual(
+      concurrentEdits.edits,
+      [
+        SourceEdit(offset: 0, length: 1, replacementLength: 25)
+      ]
+    )
+  }
+}