swiftlang · ahoppen · Mar 24, 2023 · Mar 19, 2023 · Mar 21, 2023 · Mar 14, 2023
@@ -2,8 +2,8 @@
 .build/
 # Ignore generated Xcode projects
 *.xcodeproj
-# Ignore Data Xcode stores when opening the project directly
-/.swiftpm/xcode
+# Ignore SwiftPM state, such as the generated xcodeproj when opening the project directly
+.swiftpm
 # Ignore user state in Xcode Projects
 *.xcuserdatad
 UserInterfaceState.xcuserstate

@@ -9,9 +9,7 @@
     "rules": {
         "AlwaysUseLowerCamelCase": false,
         "AmbiguousTrailingClosureOverload": false,
-        "DontRepeatTypeInStaticProperties": false,
         "NoBlockComments": false,
-        "Spacing": false,
         "UseLetInEveryBoundCaseVariable": false,
         "UseSynthesizedInitializer": false
     }

@@ -23,6 +23,9 @@ let package = Package(
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
         "SyntaxSupport",
         "Utils",
+      ],
+      exclude: [
+        "templates/swiftsyntax/SwiftSyntaxDoccIndexTemplate.md"
       ]
     ),
     .target(

@@ -2,18 +2,18 @@
 
 This directory contains file to generate source code that is part of the SwiftSyntax package. If you are looking to generate Swift code yourself, you might be interested in [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder).
 
-SwiftSyntax currently uses two mechanisms to generate source code: gyb and SwiftSyntaxBuilder. Both can be run using 
-
-```bash
-./build-script.py generate-source-code --toolchain /path/to/toolchain.xctoolchain/usr
-```
+Some source code inside SwiftSyntax is generated using [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder), a Swift library whose purpose is to generate Swift code using Swift itself. This kind of code generation is performed by the Swift package defined in this directory.
 
-## `gyb`
+This directory is a standalone package that uses a pinned version of SwiftSyntaxBuilder. It is thus NOT using SwiftSyntaxBuilder of the parent directory. This guarantees that when `generate-swiftsyntax` is run, it can't break its own build.
 
-Files generated using `gyb` have a `.gyb` suffix. [ChangingSwiftSyntax.md](../Sources/SwiftSyntax/Documentation.docc/ChangingSwiftSyntax.md) contains documentation how `gyb` works. This directory contains the definitions that back most of gyb-generation in `gyb_syntax_support`.
+To re-generate the files after changing `CodeGeneration` run the `generate-swiftsyntax` 
+target of `CodeGeneration` and pass `path/to/swift-syntax/Sources` as the argument.
 
-## `SwiftSyntaxBuilder`
+On the command line, this would be
+```bash
+swift run --package-path CodeGeneration generate-swiftsyntax Sources
+```
 
-Some source code inside SwiftSyntax is generated using [SwiftSyntaxBuilder](../Sources/SwiftSyntaxBuilder), a Swift library whose purpose is to generate Swift code using Swift itself. This kind of code generation is performed by the Swift package defined in this directory.
+Or if you open the `CodeGeneration` package in Xcode, you can add the 
+`generate-swift syntax ../Sources` arguments using Product -> Scheme -> Edit Scheme…
 
-This directory is a standalone package that uses a pinned version of SwiftSyntaxBuilder. It is thus NOT using SwiftSyntaxBuilder of the parent directory. This guarantees that when `generate-swiftsyntaxbuilder` is run, it can't break its own build.
@@ -12,20 +12,19 @@
 
 import SwiftSyntax
 
-public func generateCopyrightHeader(for generator: String) -> Trivia {
-  return """
-    //// Automatically Generated by \(generator)
-    //// Do Not Edit Directly!
-    //===----------------------------------------------------------------------===//
-    //
-    // This source file is part of the Swift.org open source project
-    //
-    // Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
-    // Licensed under Apache License v2.0 with Runtime Library Exception
-    //
-    // See https://swift.org/LICENSE.txt for license information
-    // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-    //
-    //===----------------------------------------------------------------------===//
-    """ + .newlines(2)
-}
+public var copyrightHeader: Trivia =
+  """
+  //// Automatically generated by generate-swiftsyntax
+  //// Do not edit directly!
+  //===----------------------------------------------------------------------===//
+  //
+  // This source file is part of the Swift.org open source project
+  //
+  // Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+  // Licensed under Apache License v2.0 with Runtime Library Exception
+  //
+  // See https://swift.org/LICENSE.txt for license information
+  // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+  //
+  //===----------------------------------------------------------------------===//
+  """ + .newlines(2)
@@ -26,7 +26,7 @@ public struct SyntaxBuildableType: Hashable {
   public init(syntaxKind: String, isOptional: Bool = false) {
     self.isOptional = isOptional
     if syntaxKind.hasSuffix("Token") {
-      // There are different token kinds but all of them are represented by `Token` in the Swift source (see `kind_to_type` in `gyb_syntax_support`).
+      // There are different token kinds but all of them are represented by `Token` in the Swift source (see `kindToType` in `SyntaxSupport/Utils.swift`).
       self.syntaxKind = "Token"
       self.tokenKind = syntaxKind
     } else {

@@ -22,7 +22,7 @@ public func dedented(string: String) -> String {
 }
 
 /// Creates a single-line documentation string from indented
-/// documentation as written in `gyb_syntax_support`.
+/// documentation as written in `CodeGeneration`.
 public func flattened(indentedDocumentation: String) -> String {
   dedented(string: indentedDocumentation)
     .replacingOccurrences(of: "\n", with: "")

@@ -74,6 +74,8 @@ struct GenerateSwiftSyntax: ParsableCommand {
   var verbose: Bool = false
 
   func run() throws {
+    let destination = URL(fileURLWithPath: self.destination).standardizedFileURL
+
     let fileSpecs: [GeneratedFileSpec] =
       [
         // SwiftBasicFormat
@@ -123,21 +125,23 @@ struct GenerateSwiftSyntax: ParsableCommand {
     let previouslyGeneratedFilesLock = NSLock()
     var previouslyGeneratedFiles = Set(
       modules.flatMap { (module) -> [URL] in
-        let generatedDir = URL(fileURLWithPath: destination)
+        let generatedDir =
+          destination
           .appendingPathComponent(module)
           .appendingPathComponent("generated")
         return FileManager.default
           .enumerator(at: generatedDir, includingPropertiesForKeys: nil)!
           .compactMap { $0 as? URL }
           .filter { !$0.hasDirectoryPath }
+          .map { $0.resolvingSymlinksInPath() }
       }
     )
 
     var errors: [Error] = []
     DispatchQueue.concurrentPerform(iterations: fileSpecs.count) { index in
       let fileSpec = fileSpecs[index]
       do {
-        var destination = URL(fileURLWithPath: destination)
+        var destination = destination
         for component in fileSpec.pathComponents {
           destination = destination.appendingPathComponent(component)
         }

@@ -49,7 +49,7 @@ extension Child {
   }
 }
 
-let basicFormatFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftbasicformat")) {
+let basicFormatFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("import SwiftSyntax")
 
   try! ClassDeclSyntax("open class BasicFormat: SyntaxRewriter") {

@@ -29,7 +29,7 @@ fileprivate var node_child_classifications: [ChildClassification] {
   return result
 }
 
-let syntaxClassificationFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-ideutils")) {
+let syntaxClassificationFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! EnumDeclSyntax("public enum SyntaxClassification") {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let declarationModifierFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftparser")) {
+let declarationModifierFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! EnumDeclSyntax("enum DeclarationModifier: TokenSpecSet") {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let parserEntryFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftparser")) {
+let parserEntryFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension Parser") {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftparser")) {
+let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension TokenSpec") {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let typeAttributeFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftparser")) {
+let typeAttributeFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension Parser") {

@@ -21,7 +21,7 @@ let lookupTable = ArrayExprSyntax(leftSquare: .leftSquareBracketToken(trailingTr
   }
 }
 
-let keywordFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let keywordFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax(
     """
     /// Make `StaticString` equatable so we can use it as the raw value for Keyword.

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let miscFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let miscFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ExtensionDeclSyntax("extension Syntax") {
     try VariableDeclSyntax("public static var structure: SyntaxNodeStructure") {
       let choices = ArrayExprSyntax {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   for node in SYNTAX_NODES where node.isBase {
     DeclSyntax(
       """

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try FunctionDeclSyntax(
     """
     /// Check that the `layout` is valid for the given 'SyntaxKind'.

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxAnyVisitorFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxAnyVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ClassDeclSyntax(
     """
     /// A `SyntaxVisitor` that can visit the nodes as generic `Syntax` values.

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxBaseNodesFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxBaseNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   for node in SYNTAX_NODES where node.isBase {
     DeclSyntax(
       """

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxCollectionsFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxCollectionsFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax(
     """
     public protocol SyntaxCollection: SyntaxProtocol, Sequence where Element: SyntaxProtocol {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxEnumFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxEnumFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! EnumDeclSyntax(
     """
     /// Enum to exhaustively switch over all different syntax nodes.

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxKindFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! EnumDeclSyntax(
     """
     /// Enumerates the known kinds of Syntax represented in the Syntax tree.

@@ -15,12 +15,12 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-/// This gyb-file generates the syntax nodes for SwiftSyntax. To keep the generated
+/// This file generates the syntax nodes for SwiftSyntax. To keep the generated
 /// files at a managable file size, it is to be invoked multiple times with the
 /// variable `emitKind` set to a base kind listed in
 /// It then only emits those syntax nodes whose base kind are that specified kind.
 func syntaxNode(emitKind: String) -> SourceFileSyntax {
-  SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+  SourceFileSyntax(leadingTrivia: copyrightHeader) {
     for node in SYNTAX_NODES where !node.isBase && node.collectionElement.isEmpty && node.baseKind == emitKind {
       // We are actually handling this node now
       let nodeDoc = node.description.map { "/// \($0)" }

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxRewriterFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ClassDeclSyntax(
     """
     //

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxTraitsFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxTraitsFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   for trait in TRAITS {
     try! ProtocolDeclSyntax(
       """

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxTransformFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxTransformFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ProtocolDeclSyntax("public protocol SyntaxTransformVisitor") {
     DeclSyntax("associatedtype ResultType = Void")
 

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let syntaxVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax(
     """
     /// The enum describes how the SyntaxVistor should continue after visiting

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let tokenKindFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let tokenKindFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! EnumDeclSyntax(
     """
     /// Enumerates the kinds of tokens in the Swift language.

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let tokensFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let tokensFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   try! ExtensionDeclSyntax("extension TokenSyntax") {
     for token in SYNTAX_TOKENS {
       if token.isKeyword {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let triviaFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntax")) {
+let triviaFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax(
     """
     public enum TriviaPosition {

@@ -16,7 +16,7 @@ import SyntaxSupport
 import Utils
 import SwiftBasicFormat
 
-let buildableCollectionNodesFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntaxbuilder")) {
+let buildableCollectionNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("import SwiftSyntax")
 
   for node in SYNTAX_NODES where node.isSyntaxCollection {

@@ -15,7 +15,7 @@ import SwiftSyntaxBuilder
 import SyntaxSupport
 import Utils
 
-let buildableNodesFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntaxbuilder")) {
+let buildableNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("import SwiftSyntax")
 
   for node in SYNTAX_NODES where node.isBuildable {

@@ -15,7 +15,7 @@ import SyntaxSupport
 import SwiftSyntaxBuilder
 import Utils
 
-let resultBuildersFile = SourceFileSyntax(leadingTrivia: generateCopyrightHeader(for: "generate-swiftsyntaxbuilder")) {
+let resultBuildersFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   DeclSyntax("import SwiftSyntax")
 
   for node in SYNTAX_NODES where node.isSyntaxCollection {