WIP: Rename firstPotentialTokenKind to firstPotentialTokenChoices

Add logic for `firstPotentialTokenChoices` of base nodes and unexpected nodes

Author: StevenWong12

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -129,7 +129,7 @@ public class Node {
         }
 
         choices.append(contentsOf: getChildTokenChoice(child))
-        
+
         if !child.isOptional {
           break
         }
@@ -146,6 +146,11 @@ public class Node {
       }.flatMap { $0 }
     }
 
+    if self.kind.isBase {
+      return []
+    }
+
+    assertionFailure("should not match this case")
     return []
   }
 

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/IncrementalParseUtilsFile.swift

@@ -27,7 +27,7 @@ let IncrementalParseUtilsFile = SourceFileSyntax(leadingTrivia: copyrightHeader)
       """
         /// This property presents the first potential ``TokenSpec``s of all ``SyntaxKind``s
         @_spi(IncrementalParse)
-        public var firstPotentialTokenKind: [TokenSpec]
+        public var firstPotentialTokenChoices: [TokenSpec]
       """
     ) {
       try SwitchExprSyntax("switch self") {
@@ -93,12 +93,38 @@ let IncrementalParseUtilsFile = SourceFileSyntax(leadingTrivia: copyrightHeader)
             return []
           }
 
-          return self.kind.syntaxLayout[nextPotentialNodeIndex].flatMap { $0.firstPotentialTokenKind }
+          return self.kind.syntaxLayout[nextPotentialNodeIndex].flatMap { $0.firstPotentialTokenChoices }
 
         }
         return []
       }
       """
     )
+
+    DeclSyntax(
+      """
+      public var isSelfCompleted: Bool {
+        if let children = self.layoutView?.children,
+           let lastNonNilChildIdx = children.lastIndex(where: {
+             $0 != nil
+           }),
+           lastNonNilChildIdx != children.endIndex
+        {
+
+          let nextPotentialNodeIndex = children.index(after: lastNonNilChildIdx)
+
+          if self.kind.syntaxLayout[nextPotentialNodeIndex].allSatisfy({
+            $0 == .unexpectedNodes
+          }) {
+            return nextPotentialNodeIndex == children.index(before: children.endIndex)
+          } else {
+            return children.index(after: nextPotentialNodeIndex) == children.index(before: children.endIndex)
+          }
+        }
+
+        return true
+      }
+      """
+    )
   }
 }

Sources/SwiftParser/Parser.swift

@@ -652,15 +652,21 @@ extension Parser {
   func nodeCanBeReused(node: Syntax) -> Bool {
     var underlyingRaw = node.raw
 
+    if let codeBlockItem = node.as(CodeBlockItemSyntax.self) {
+      underlyingRaw = codeBlockItem.item.raw
+    }
+    // Since `firstPotentialTokenChoices` of an unexpected node is empty too, we need to verify whether the node to be reused is completed, if so, we just break.
+    // See comments below.
+    if underlyingRaw.isSelfCompleted {
+      return true
+    }
+
     var lookahead = self.lookahead()
     lookahead.lexemes.advance(by: node.byteSize, currentToken: &lookahead.currentToken)
 
     let currentTokenSpec = PrepareForKeywordMatch(lookahead.currentToken)
 
-    if let codeBlockItem = node.as(CodeBlockItemSyntax.self) {
-      underlyingRaw = codeBlockItem.item.raw
-    }
-
-    return underlyingRaw.nextPotentialTokenChoices.allSatisfy { !($0 ~= currentTokenSpec) }
+    // If `nextPotentialTokenChoices` is empty, we presume next potential child is a base node and reject this node to be reused. Since almost all tokens can be the introducer of a base node.
+    return underlyingRaw.nextPotentialTokenChoices.allSatisfy { !($0 ~= currentTokenSpec) } && !underlyingRaw.nextPotentialTokenChoices.isEmpty
   }
 }

Sources/SwiftParser/generated/IncrementalParseUtils.swift

@@ -17,7 +17,7 @@
 extension SyntaxKind {
   /// This property presents the first potential ``TokenSpec``s of all ``SyntaxKind``s
   @_spi(IncrementalParse)
-  public var firstPotentialTokenKind: [TokenSpec] {
+  public var firstPotentialTokenChoices: [TokenSpec] {
     switch self {
     case .accessorBlock:
       return [.leftBrace]
@@ -7682,10 +7682,32 @@ extension RawSyntax {
       }
 
       return self.kind.syntaxLayout[nextPotentialNodeIndex].flatMap {
-        $0.firstPotentialTokenKind
+        $0.firstPotentialTokenChoices
       }
 
     }
     return []
   }
+  
+  public var isSelfCompleted: Bool {
+    if let children = self.layoutView?.children,
+       let lastNonNilChildIdx = children.lastIndex(where: {
+         $0 != nil
+       }),
+       lastNonNilChildIdx != children.endIndex
+    {
+
+      let nextPotentialNodeIndex = children.index(after: lastNonNilChildIdx)
+
+      if self.kind.syntaxLayout[nextPotentialNodeIndex].allSatisfy({
+        $0 == .unexpectedNodes
+      }) {
+        return nextPotentialNodeIndex == children.index(before: children.endIndex)
+      } else {
+        return children.index(after: nextPotentialNodeIndex) == children.index(before: children.endIndex)
+      }
+    }
+
+    return true
+  }
 }