Enable a mode in the parser in which it inspects alternative token choices to mutate test cases

The basic idea is that the parser records which `TokenSpec`s it checked for at individual offsets in the source. We can then use that information to generate new, interesting test cases by replacing a token by the one of the `TokenSpec` we checked for. This technique has found 11 bugs in the parser and I’m expecting it to find quite a few more once we assert that tokens have one of the expected kinds.

Gathering of that information is hidden behind a conditional compilation flag because just performing the check of whether we want to record alternative token choices inflicts a 6% performance regression, which doesn’t provide any value except when we are running SwiftParserTest.

Author: ahoppen

Sources/SwiftParser/Lexer/LexemeSequence.swift

@@ -94,6 +94,20 @@ extension Lexer {
         return remainingText
       }
     }
+
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    /// If `pointer` is in the source buffer of this `LexemeSequence`, return
+    /// its offset, otherwise `nil`. Should only be used for the parser's
+    /// alternate token introspection
+    func offset(of pointer: UnsafePointer<UInt8>) -> Int? {
+      let offset = pointer - self.sourceBufferStart.input.baseAddress!
+      if offset <= self.sourceBufferStart.input.count {
+        return offset
+      } else {
+        return nil
+      }
+    }
+    #endif
   }
 
   @_spi(RawSyntax)

Sources/SwiftParser/Lookahead.swift

@@ -70,6 +70,12 @@ extension Parser.Lookahead: TokenConsumer {
   mutating func eat(_ spec: TokenSpec) -> Token {
     return self.consume(if: spec)!
   }
+
+  #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+  var shouldRecordAlternativeTokenChoices: Bool { false }
+
+  mutating public func recordAlternativeTokenChoice(for lexeme: Lexer.Lexeme, choices: [TokenSpec]) {}
+  #endif
 }
 
 extension Parser.Lookahead {

Sources/SwiftParser/Parser.swift

@@ -207,6 +207,34 @@ public struct Parser {
       break
     }
   }
+
+  #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+  var shouldRecordAlternativeTokenChoices: Bool = false
+
+  public mutating func enableAlternativeTokenChoices() {
+    shouldRecordAlternativeTokenChoices = true
+  }
+
+  /// When compiled with `PARSER_ALTERNATE_TOKEN_INTROSPECTION`, and
+  /// `shouldRecordAlternativeTokenChoices` is `true` the parser records which
+  /// `TokenSpec`s it checked for a token at a specific offset in the source
+  /// file. The offsets are the location of the token text's start (excluding
+  /// leading trivia).
+  ///
+  /// This information allows testing techniques to replace tokens by these
+  /// alternate token choices to generate new, interesting test cases
+  @_spi(RawSyntax)
+  public var alternativeTokenChoices: [Int: [TokenSpec]] = [:]
+
+  mutating func recordAlternativeTokenChoice(for lexeme: Lexer.Lexeme, choices: [TokenSpec]) {
+    guard let lexemeBaseAddress = lexeme.tokenText.baseAddress,
+      let offset = lexemes.offset(of: lexemeBaseAddress)
+    else {
+      return
+    }
+    alternativeTokenChoices[offset, default: []].append(contentsOf: choices)
+  }
+  #endif
 }
 
 // MARK: Inspecting Tokens

Sources/SwiftParser/Recovery.swift

@@ -67,6 +67,11 @@ extension Parser.Lookahead {
     _ spec2: TokenSpec,
     _ spec3: TokenSpec
   ) -> RecoveryConsumptionHandle? {
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    if shouldRecordAlternativeTokenChoices {
+      recordAlternativeTokenChoice(for: self.currentToken, choices: [spec1, spec2, spec3])
+    }
+    #endif
     let initialTokensConsumed = self.tokensConsumed
 
     let recoveryPrecedence = min(spec1.recoveryPrecedence, spec2.recoveryPrecedence, spec3.recoveryPrecedence)
@@ -118,6 +123,11 @@ extension Parser.Lookahead {
     anyIn specSet: SpecSet.Type,
     overrideRecoveryPrecedence: TokenPrecedence? = nil
   ) -> (match: SpecSet, handle: RecoveryConsumptionHandle)? {
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    if shouldRecordAlternativeTokenChoices {
+      recordAlternativeTokenChoice(for: self.currentToken, choices: specSet.allCases.map(\.spec))
+    }
+    #endif
     let initialTokensConsumed = self.tokensConsumed
 
     assert(!specSet.allCases.isEmpty, "SpecSet must have at least one case")

Sources/SwiftParser/TokenConsumer.swift

@@ -13,8 +13,7 @@
 @_spi(RawSyntax) import SwiftSyntax
 
 /// A type that consumes  instances of `TokenSyntax`.
-@_spi(RawSyntax)
-public protocol TokenConsumer {
+protocol TokenConsumer {
   associatedtype Token
   /// The current token syntax being examined by the consumer
   var currentToken: Lexer.Lexeme { get }
@@ -32,6 +31,21 @@ public protocol TokenConsumer {
   func peek() -> Lexer.Lexeme
 
   func lookahead() -> Parser.Lookahead
+
+  #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+  var shouldRecordAlternativeTokenChoices: Bool { get }
+
+  /// When compiled with `PARSER_ALTERNATE_TOKEN_INTROSPECTION`, record
+  /// alternative tokens that the parser was looking for at the offset of
+  /// `lexeme`.
+  ///
+  /// E.g. if at offset 33, we issue an `at(.leftParen)` call, this will record
+  /// that `.leftParen` is an interesting token at offset 33. This allows the
+  /// test case mutators to prefer replacing the current token at offset 33 by a
+  /// left paren, because apparently this would be a code path that the parser
+  /// is interested in.
+  mutating func recordAlternativeTokenChoice(for lexeme: Lexer.Lexeme, choices: [TokenSpec])
+  #endif
 }
 
 // MARK: Checking if we are at one specific token (`at`)
@@ -51,6 +65,11 @@ extension TokenConsumer {
   /// Returns whether the the current token matches `spec`
   @inline(__always)
   mutating func at(_ spec: TokenSpec) -> Bool {
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    if shouldRecordAlternativeTokenChoices {
+      recordAlternativeTokenChoice(for: self.currentToken, choices: [spec])
+    }
+    #endif
     return spec ~= self.currentToken
   }
 
@@ -60,6 +79,11 @@ extension TokenConsumer {
     _ spec1: TokenSpec,
     _ spec2: TokenSpec
   ) -> Bool {
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    if shouldRecordAlternativeTokenChoices {
+      recordAlternativeTokenChoice(for: self.currentToken, choices: [spec1, spec2])
+    }
+    #endif
     switch self.currentToken {
     case spec1: return true
     case spec2: return true
@@ -74,6 +98,11 @@ extension TokenConsumer {
     _ spec2: TokenSpec,
     _ spec3: TokenSpec
   ) -> Bool {
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    if shouldRecordAlternativeTokenChoices {
+      recordAlternativeTokenChoice(for: self.currentToken, choices: [spec1, spec2, spec3])
+    }
+    #endif
     switch self.currentToken {
     case spec1: return true
     case spec2: return true
@@ -93,6 +122,11 @@ extension TokenConsumer {
   /// as well as a handle to consume that token.
   @inline(__always)
   mutating func at<SpecSet: TokenSpecSet>(anyIn specSet: SpecSet.Type) -> (SpecSet, TokenConsumptionHandle)? {
+    #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+    if shouldRecordAlternativeTokenChoices {
+      recordAlternativeTokenChoice(for: self.currentToken, choices: specSet.allCases.map(\.spec))
+    }
+    #endif
     if let matchedKind = SpecSet(lexeme: self.currentToken) {
       assert(matchedKind.spec ~= self.currentToken)
       return (

Sources/SwiftParser/TokenSpec.swift

@@ -44,7 +44,8 @@ struct PrepareForKeywordMatch {
 /// marked `@inline(__always)` so the compiler inlines the `RawTokenKind` we are
 /// matching against and is thus able to rule out one of the branches in
 /// `matches(rawTokenKind:text:)` based on the matched kind.
-struct TokenSpec {
+@_spi(RawSyntax)
+public struct TokenSpec {
   /// The kind we expect the token that we want to consume to have.
   /// This can be a keyword, in which case the `TokenSpec` will also match an
   /// identifier with the same text as the keyword and remap it to that keyword
@@ -160,6 +161,28 @@ struct TokenSpec {
       atStartOfLine: lexeme.isAtStartOfLine
     )
   }
+
+  /// Returns a `TokenKind` that will most likely be parsed as a token that
+  /// matches this `TokenSpec`.
+  ///
+  /// IMPORTANT: Should only be used when generating tokens during the
+  /// modification of test cases. This should never be used in the parser itself.
+  public var synthesizedTokenKind: TokenKind {
+    switch rawTokenKind {
+    case .integerLiteral: return .integerLiteral("1")
+    case .floatingLiteral: return .floatingLiteral("1.0")
+    case .regexLiteral: return .regexLiteral("/.*/")
+    case .identifier: return .identifier("myIdent")
+    case .binaryOperator: return .binaryOperator("+")
+    case .postfixOperator: return .postfixOperator("++")
+    case .prefixOperator: return .prefixOperator("!")
+    case .dollarIdentifier: return .dollarIdentifier("$0")
+    case .rawStringDelimiter: return .rawStringDelimiter("#")
+    case .stringSegment: return .stringSegment("abc")
+    case .keyword: return .keyword(keyword!)
+    default: return TokenKind.fromRaw(kind: rawTokenKind, text: "")
+    }
+  }
 }
 
 extension TokenConsumer {

Sources/SwiftParser/generated/TokenSpecStaticMembers.swift

@@ -15,7 +15,8 @@
 
 import SwiftSyntax
 
-extension TokenSpec {
+@_spi(RawSyntax)
+public extension TokenSpec {
   static var eof: TokenSpec { 
     return TokenSpec(.eof) 
   }

Tests/SwiftParserTest/Assertions.swift

@@ -446,6 +446,39 @@ func AssertDiagnostic<T: SyntaxProtocol>(
   }
 }
 
+class MutatedTreePrinter: SyntaxVisitor {
+  private var mutations: [Int: TokenSpec] = [:]
+  private var printedSource: [UInt8] = []
+
+  /// Prints `tree` by replacing the tokens whose offset is in `mutations` by
+  /// a token that matches the corresponding `TokenSpec`.
+  static func print(tree: Syntax, mutations: [Int: TokenSpec]) -> [UInt8] {
+    let printer = MutatedTreePrinter(mutations: mutations)
+    printer.walk(tree)
+    return printer.printedSource
+  }
+
+  private init(mutations: [Int: TokenSpec]) {
+    self.mutations = mutations
+    super.init(viewMode: .sourceAccurate)
+  }
+
+  override func visit(_ node: TokenSyntax) -> SyntaxVisitorContinueKind {
+    if let mutation = mutations[node.positionAfterSkippingLeadingTrivia.utf8Offset] {
+      let token = TokenSyntax(
+        mutation.synthesizedTokenKind,
+        leadingTrivia: node.leadingTrivia,
+        trailingTrivia: node.trailingTrivia,
+        presence: .present
+      )
+      printedSource.append(contentsOf: token.syntaxTextBytes)
+      return .skipChildren
+    }
+    printedSource.append(contentsOf: node.syntaxTextBytes)
+    return .skipChildren
+  }
+}
+
 public struct AssertParseOptions: OptionSet {
   public var rawValue: UInt8
 
@@ -489,38 +522,6 @@ func AssertParse(
   )
 }
 
-/// Same as `AssertParse` overload with a `(String) -> S` `parse`,
-/// constructing a `Parser` from the given `String` and passing that to
-/// `parse` instead.
-func AssertParse<S: SyntaxProtocol>(
-  _ markedSource: String,
-  _ parse: (inout Parser) -> S,
-  substructure expectedSubstructure: Syntax? = nil,
-  substructureAfterMarker: String = "START",
-  diagnostics expectedDiagnostics: [DiagnosticSpec] = [],
-  applyFixIts: [String]? = nil,
-  fixedSource expectedFixedSource: String? = nil,
-  options: AssertParseOptions = [],
-  file: StaticString = #file,
-  line: UInt = #line
-) {
-  AssertParse(
-    markedSource,
-    { (source: String) -> S in
-      var parser = Parser(source)
-      return parse(&parser)
-    },
-    substructure: expectedSubstructure,
-    substructureAfterMarker: substructureAfterMarker,
-    diagnostics: expectedDiagnostics,
-    applyFixIts: applyFixIts,
-    fixedSource: expectedFixedSource,
-    options: options,
-    file: file,
-    line: line
-  )
-}
-
 /// Removes any test markers from `markedSource` (1) and parses the result
 /// using `parse`. By default it only checks if the parsed syntax tree is
 /// printable back to the origin source, ie. it round trips.
@@ -541,7 +542,7 @@ func AssertParse<S: SyntaxProtocol>(
 ///     this string.
 func AssertParse<S: SyntaxProtocol>(
   _ markedSource: String,
-  _ parse: (String) -> S,
+  _ parse: (inout Parser) -> S,
   substructure expectedSubstructure: Syntax? = nil,
   substructureAfterMarker: String = "START",
   diagnostics expectedDiagnostics: [DiagnosticSpec] = [],
@@ -555,7 +556,18 @@ func AssertParse<S: SyntaxProtocol>(
   var (markerLocations, source) = extractMarkers(markedSource)
   markerLocations["START"] = 0
 
-  let tree: S = parse(source)
+  let enableTestCaseMutation = false
+  // FIXME: Currently, tests are failing when we enable test case mutation.
+  // Once all of those issues are fixed, this should become
+  // let enableTestCaseMutation = ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"] != "1"
+
+  var parser = Parser(source)
+  #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+  if enableTestCaseMutation {
+    parser.enableAlternativeTokenChoices()
+  }
+  #endif
+  let tree: S = parse(&parser)
 
   // Round-trip
   AssertStringsEqualWithDiff(
@@ -615,4 +627,38 @@ func AssertParse<S: SyntaxProtocol>(
       line: line
     )
   }
+
+  #if PARSER_ALTERNATE_TOKEN_INTROSPECTION
+  if enableTestCaseMutation {
+    let mutations: [(offset: Int, replacement: TokenSpec)] = parser.alternativeTokenChoices.flatMap { offset, replacements in
+      return replacements.map { (offset, $0) }
+    }
+    DispatchQueue.concurrentPerform(iterations: mutations.count) { index in
+      let mutation = mutations[index]
+      let alternateSource = MutatedTreePrinter.print(tree: Syntax(tree), mutations: [mutation.offset: mutation.replacement])
+      alternateSource.withUnsafeBufferPointer { buf in
+        let mutatedSource = String(decoding: buf, as: UTF8.self)
+        // Check that we don't hit any assertions in the parser while parsing
+        // the mutated source and that it round-trips
+        var mutatedParser = Parser(buf)
+        let mutatedTree = parse(&mutatedParser)
+        AssertStringsEqualWithDiff(
+          "\(mutatedTree)",
+          mutatedSource,
+          additionalInfo: """
+            Mutated source failed to round-trip.
+
+            Mutated source:
+            \(mutatedSource)
+
+            Actual syntax tree:
+            \(mutatedTree.recursiveDescription)
+            """,
+          file: file,
+          line: line
+        )
+      }
+    }
+  }
+  #endif
 }

Tests/SwiftParserTest/Parser+EntryTests.swift

@@ -16,18 +16,20 @@ import XCTest
 
 public class EntryTests: XCTestCase {
   func testTopLevelStringParse() throws {
-    AssertParse("func test() {}", { Parser.parse(source: $0) })
+    let source = "func test() {}"
+    let tree = Parser.parse(source: source)
+    XCTAssert(tree.is(SourceFileSyntax.self))
+    XCTAssert(!tree.hasError)
+    XCTAssertEqual(tree.description, source)
   }
 
   func testTopLevelBufferParse() throws {
-    AssertParse(
-      "func test() {}",
-      { (source: String) -> SourceFileSyntax in
-        var source = source
-        source.makeContiguousUTF8()
-        return source.withUTF8 { Parser.parse(source: $0) }
-      }
-    )
+    var source = "func test() {}"
+    source.makeContiguousUTF8()
+    let tree = source.withUTF8 { Parser.parse(source: $0) }
+    XCTAssert(tree.is(SourceFileSyntax.self))
+    XCTAssert(!tree.hasError)
+    XCTAssertEqual(tree.description, source)
   }
 
   func testSyntaxParse() throws {