From a96f62fd089c65bffd0c2a1352c0b51579c02849 Mon Sep 17 00:00:00 2001 From: Mikhail Tishin Date: Thu, 28 Sep 2023 22:09:03 +0300 Subject: [PATCH 1/2] Fix process argument limit error when building on Windows by merging generated sources into a single file --- Generator/Generator/BuiltinGen.swift | 8 +++--- Generator/Generator/ClassGen.swift | 32 +++++++++++++++--------- Generator/Generator/UtilityGen.swift | 8 +++--- Generator/Generator/main.swift | 29 +++++++++++++-------- Plugins/CodeGeneratorPlugin/plugin.swift | 10 +++++++- 5 files changed, 58 insertions(+), 29 deletions(-) diff --git a/Generator/Generator/BuiltinGen.swift b/Generator/Generator/BuiltinGen.swift index 8b4ed41ce..988bd39ba 100644 --- a/Generator/Generator/BuiltinGen.swift +++ b/Generator/Generator/BuiltinGen.swift @@ -407,7 +407,7 @@ enum BKind { var builtinGodotTypeNames: [String:BKind] = ["Variant": .isClass] var builtinClassStorage: [String:String] = [:] -func generateBuiltinClasses (values: [JGodotBuiltinClass], outputDir: String) { +func generateBuiltinClasses (values: [JGodotBuiltinClass], outputDir: String, sharedPrinter: Printer?) { func generateBuiltinClass (p: Printer, _ bc: JGodotBuiltinClass, _ docClass: DocBuiltinClass?) { // TODO: isKeyed, hasDestrcturo, @@ -623,13 +623,15 @@ func generateBuiltinClasses (values: [JGodotBuiltinClass], outputDir: String) { case "int", "float", "bool": break default: - let p = Printer () + let p: Printer = sharedPrinter ?? Printer () p.preamble() let docClass = loadBuiltinDoc(base: docRoot, name: bc.name) mapStringToSwift = bc.name != "String" generateBuiltinClass (p: p, bc, docClass) mapStringToSwift = true - p.save(outputDir + "/\(bc.name).swift") + if sharedPrinter == nil { + p.save(outputDir + "/\(bc.name).swift") + } } } } diff --git a/Generator/Generator/ClassGen.swift b/Generator/Generator/ClassGen.swift index ec33c1510..e5beff30d 100644 --- a/Generator/Generator/ClassGen.swift +++ b/Generator/Generator/ClassGen.swift @@ -414,7 +414,7 @@ var okList = [ "RefCounted", "Node", "Sprite2D", "Node2D", "CanvasItem", "Object var okList: [String] = [] #endif -func generateClasses (values: [JGodotExtensionAPIClass], outputDir: String) { +func generateClasses (values: [JGodotExtensionAPIClass], outputDir: String, sharedPrinter: Printer?) { // TODO: duplicate, we can remove this and use classMap // Assemble all the reference types, we use to test later for cdef in values { @@ -447,19 +447,25 @@ func generateClasses (values: [JGodotExtensionAPIClass], outputDir: String) { // } // } - let semaphore = DispatchSemaphore(value: 0) + if sharedPrinter == nil { + let semaphore = DispatchSemaphore(value: 0) - let _ = Task { - await withTaskGroup(of: Void.self) { group in - for cdef in values { - group.addTask { - processClass (cdef: cdef, outputDir: outputDir) + let _ = Task { + await withTaskGroup(of: Void.self) { group in + for cdef in values { + group.addTask { + processClass (cdef: cdef, outputDir: outputDir, sharedPrinter: sharedPrinter) + } } } + semaphore.signal() + } + semaphore.wait() + } else { + for cdef in values { + processClass (cdef: cdef, outputDir: outputDir, sharedPrinter: sharedPrinter) } - semaphore.signal() } - semaphore.wait() } func generateSignalType (_ p: Printer, _ cdef: JGodotExtensionAPIClass, _ signal: JGodotSignal, _ name: String) -> String { @@ -592,7 +598,7 @@ func generateSignalDocAppendix (_ p: Printer, cdef: JGodotExtensionAPIClass, sig } } -func processClass (cdef: JGodotExtensionAPIClass, outputDir: String) { +func processClass (cdef: JGodotExtensionAPIClass, outputDir: String, sharedPrinter: Printer?) { let docClass = loadClassDoc(base: docRoot, name: cdef.name) // Determine if it is a singleton, but exclude EditorInterface @@ -600,13 +606,15 @@ func processClass (cdef: JGodotExtensionAPIClass, outputDir: String) { let asSingleton = isSingleton && cdef.name != "EditorInterface" // Clear the result - let p = Printer () + let p = sharedPrinter ?? Printer () p.preamble() p ("// Generated by Swift code generator - do not edit\n@_implementationOnly import GDExtension\n") // Save it defer { - p.save(outputDir + "\(cdef.name).swift") + if sharedPrinter == nil { + p.save(outputDir + "\(cdef.name).swift") + } } let inherits = cdef.inherits ?? "Wrapped" diff --git a/Generator/Generator/UtilityGen.swift b/Generator/Generator/UtilityGen.swift index bc20ceeed..1a60d1f40 100644 --- a/Generator/Generator/UtilityGen.swift +++ b/Generator/Generator/UtilityGen.swift @@ -7,11 +7,13 @@ import Foundation -func generateUtility(values: [JGodotUtilityFunction], outputDir: String) { - let p = Printer () +func generateUtility(values: [JGodotUtilityFunction], outputDir: String, sharedPrinter: Printer?) { + let p = sharedPrinter ?? Printer () p.preamble() defer { - p.save (outputDir + "utility.swift") + if sharedPrinter == nil { + p.save (outputDir + "utility.swift") + } } let docClass = loadClassDoc(base: docRoot, name: "@GlobalScope") diff --git a/Generator/Generator/main.swift b/Generator/Generator/main.swift index 6278199d3..4408c86fa 100644 --- a/Generator/Generator/main.swift +++ b/Generator/Generator/main.swift @@ -7,9 +7,6 @@ // import Foundation -// IF we want a single file, or one file per type -var singleFile = true - var args = CommandLine.arguments let jsonFile = args.count > 1 ? args [1] : "/Users/miguel/cvs/godot-master/extension_api.json" @@ -18,6 +15,9 @@ var docRoot = args.count > 3 ? args [3] : "/Users/miguel/cvs/godot-master/doc" let outputDir = args.count > 2 ? args [2] : generatorOutput +// IF we want a single file, or one file per type +var singleFile = args.contains("--singlefile") + if args.count < 2 { print ("Usage is: generator path-to-extension-api output-directory doc-directory") print ("- path-to-extensiona-ppi is the full path to extension_api.json from Godot") @@ -55,7 +55,8 @@ func dropMatchingPrefix (_ enumName: String, _ enumKey: String) -> String { var globalEnums: [String: JGodotGlobalEnumElement] = [:] -var coreDefPrinter = Printer() +let sharedPrinter: Printer? = singleFile ? Printer() : nil +var coreDefPrinter = sharedPrinter ?? Printer() coreDefPrinter.preamble() print ("Running with projectDir=$(projectDir) and output=\(outputDir)") @@ -89,14 +90,22 @@ for cs in jsonApi.builtinClassSizes { let generatedBuiltinDir = outputDir + "/generated-builtin/" let generatedDir = outputDir + "/generated/" -try! FileManager.default.createDirectory(atPath: generatedBuiltinDir, withIntermediateDirectories: true) -try! FileManager.default.createDirectory(atPath: generatedDir, withIntermediateDirectories: true) +if singleFile { + try! FileManager.default.createDirectory(atPath: outputDir, withIntermediateDirectories: true) +} else { + try! FileManager.default.createDirectory(atPath: generatedBuiltinDir, withIntermediateDirectories: true) + try! FileManager.default.createDirectory(atPath: generatedDir, withIntermediateDirectories: true) +} -generateBuiltinClasses(values: jsonApi.builtinClasses, outputDir: generatedBuiltinDir) -generateUtility(values: jsonApi.utilityFunctions, outputDir: generatedBuiltinDir) -generateClasses (values: jsonApi.classes, outputDir: generatedDir) +generateBuiltinClasses(values: jsonApi.builtinClasses, outputDir: generatedBuiltinDir, sharedPrinter: sharedPrinter) +generateUtility(values: jsonApi.utilityFunctions, outputDir: generatedBuiltinDir, sharedPrinter: sharedPrinter) +generateClasses (values: jsonApi.classes, outputDir: generatedDir, sharedPrinter: sharedPrinter) generateCtorPointers (coreDefPrinter) -coreDefPrinter.save (generatedBuiltinDir + "/core-defs.swift") +if let sharedPrinter { + sharedPrinter.save(outputDir + "/generated.swift") +} else { + coreDefPrinter.save (generatedBuiltinDir + "/core-defs.swift") +} //print ("Done") diff --git a/Plugins/CodeGeneratorPlugin/plugin.swift b/Plugins/CodeGeneratorPlugin/plugin.swift index 4ded056d4..92a65267a 100644 --- a/Plugins/CodeGeneratorPlugin/plugin.swift +++ b/Plugins/CodeGeneratorPlugin/plugin.swift @@ -19,14 +19,22 @@ import PackagePlugin let api = context.package.directory.appending(["Sources", "SwiftGodot", "extension_api.json"]) + var arguments: [CustomStringConvertible] = [ api, genSourcesDir ] var outputFiles: [Path] = [] + #if os(Windows) + // Windows has 32K limit on CreateProcess argument length, SPM currently doesn't handle it well + outputFiles.append(genSourcesDir.appending(subpath: "Generated.swift")) + arguments.append(context.package.directory.appending(subpath: "doc")) + arguments.append("--singlefile") + #else outputFiles.append (contentsOf: knownBuiltin.map { genSourcesDir.appending(["generated-builtin", $0])}) outputFiles.append (contentsOf: known.map { genSourcesDir.appending(["generated", $0])}) + #endif let cmd: Command = Command.buildCommand( displayName: "Generating Swift API ffrom \(api) to \(genSourcesDir)", executable: generator, - arguments: [ api, genSourcesDir ], + arguments: arguments, inputFiles: [api], outputFiles: outputFiles) From d8a270705fdea26e220c6496150841c5839fdf80 Mon Sep 17 00:00:00 2001 From: Mikhail Tishin Date: Fri, 29 Sep 2023 20:22:15 +0300 Subject: [PATCH 2/2] Add parallel generation of classes for single file mode --- Generator/Generator/BuiltinGen.swift | 6 ++--- Generator/Generator/ClassGen.swift | 28 ++++++-------------- Generator/Generator/Printer.swift | 26 ++++++++++++++++++- Generator/Generator/UtilityGen.swift | 6 ++--- Generator/Generator/main.swift | 38 +++++++++++++++------------- 5 files changed, 60 insertions(+), 44 deletions(-) diff --git a/Generator/Generator/BuiltinGen.swift b/Generator/Generator/BuiltinGen.swift index 988bd39ba..0109d5b51 100644 --- a/Generator/Generator/BuiltinGen.swift +++ b/Generator/Generator/BuiltinGen.swift @@ -407,7 +407,7 @@ enum BKind { var builtinGodotTypeNames: [String:BKind] = ["Variant": .isClass] var builtinClassStorage: [String:String] = [:] -func generateBuiltinClasses (values: [JGodotBuiltinClass], outputDir: String, sharedPrinter: Printer?) { +func generateBuiltinClasses (values: [JGodotBuiltinClass], outputDir: String?) async { func generateBuiltinClass (p: Printer, _ bc: JGodotBuiltinClass, _ docClass: DocBuiltinClass?) { // TODO: isKeyed, hasDestrcturo, @@ -623,13 +623,13 @@ func generateBuiltinClasses (values: [JGodotBuiltinClass], outputDir: String, sh case "int", "float", "bool": break default: - let p: Printer = sharedPrinter ?? Printer () + let p: Printer = await PrinterFactory.shared.initPrinter() p.preamble() let docClass = loadBuiltinDoc(base: docRoot, name: bc.name) mapStringToSwift = bc.name != "String" generateBuiltinClass (p: p, bc, docClass) mapStringToSwift = true - if sharedPrinter == nil { + if let outputDir { p.save(outputDir + "/\(bc.name).swift") } } diff --git a/Generator/Generator/ClassGen.swift b/Generator/Generator/ClassGen.swift index e5beff30d..40ccbfade 100644 --- a/Generator/Generator/ClassGen.swift +++ b/Generator/Generator/ClassGen.swift @@ -414,7 +414,7 @@ var okList = [ "RefCounted", "Node", "Sprite2D", "Node2D", "CanvasItem", "Object var okList: [String] = [] #endif -func generateClasses (values: [JGodotExtensionAPIClass], outputDir: String, sharedPrinter: Printer?) { +func generateClasses (values: [JGodotExtensionAPIClass], outputDir: String?) async { // TODO: duplicate, we can remove this and use classMap // Assemble all the reference types, we use to test later for cdef in values { @@ -447,23 +447,11 @@ func generateClasses (values: [JGodotExtensionAPIClass], outputDir: String, shar // } // } - if sharedPrinter == nil { - let semaphore = DispatchSemaphore(value: 0) - - let _ = Task { - await withTaskGroup(of: Void.self) { group in - for cdef in values { - group.addTask { - processClass (cdef: cdef, outputDir: outputDir, sharedPrinter: sharedPrinter) - } - } - } - semaphore.signal() - } - semaphore.wait() - } else { + await withTaskGroup(of: Void.self) { group in for cdef in values { - processClass (cdef: cdef, outputDir: outputDir, sharedPrinter: sharedPrinter) + group.addTask { + await processClass (cdef: cdef, outputDir: outputDir) + } } } } @@ -598,7 +586,7 @@ func generateSignalDocAppendix (_ p: Printer, cdef: JGodotExtensionAPIClass, sig } } -func processClass (cdef: JGodotExtensionAPIClass, outputDir: String, sharedPrinter: Printer?) { +func processClass (cdef: JGodotExtensionAPIClass, outputDir: String?) async { let docClass = loadClassDoc(base: docRoot, name: cdef.name) // Determine if it is a singleton, but exclude EditorInterface @@ -606,13 +594,13 @@ func processClass (cdef: JGodotExtensionAPIClass, outputDir: String, sharedPrint let asSingleton = isSingleton && cdef.name != "EditorInterface" // Clear the result - let p = sharedPrinter ?? Printer () + let p = await PrinterFactory.shared.initPrinter() p.preamble() p ("// Generated by Swift code generator - do not edit\n@_implementationOnly import GDExtension\n") // Save it defer { - if sharedPrinter == nil { + if let outputDir { p.save(outputDir + "\(cdef.name).swift") } } diff --git a/Generator/Generator/Printer.swift b/Generator/Generator/Printer.swift index 341d36fc9..dfb183ecd 100644 --- a/Generator/Generator/Printer.swift +++ b/Generator/Generator/Printer.swift @@ -7,7 +7,7 @@ import Foundation -public class Printer { +class Printer { // Where we accumulate our output for the p/b routines var result = "" var indentStr = "" // The current indentation string, based on `indent` @@ -16,6 +16,8 @@ public class Printer { indentStr = String (repeating: " ", count: indent) } } + + fileprivate init() {} func preamble () { p ("// This file is autogenerated, do not edit\n") @@ -60,3 +62,25 @@ public class Printer { try! result.write(toFile: file, atomically: true, encoding: .utf8) } } + +actor PrinterFactory { + static let shared = PrinterFactory() + + private var printers: [Printer] = [] + + func initPrinter() -> Printer { + let printer = Printer() + printers.append(printer) + return printer + } + + func save (_ file: String) { + let result = printers.map({ $0.result }).joined(separator: "\n") + if let existing = try? String (contentsOfFile: file) { + if existing == result { + return + } + } + try! result.write(toFile: file, atomically: true, encoding: .utf8) + } +} diff --git a/Generator/Generator/UtilityGen.swift b/Generator/Generator/UtilityGen.swift index 1a60d1f40..77496b2a4 100644 --- a/Generator/Generator/UtilityGen.swift +++ b/Generator/Generator/UtilityGen.swift @@ -7,11 +7,11 @@ import Foundation -func generateUtility(values: [JGodotUtilityFunction], outputDir: String, sharedPrinter: Printer?) { - let p = sharedPrinter ?? Printer () +func generateUtility(values: [JGodotUtilityFunction], outputDir: String?) async { + let p = await PrinterFactory.shared.initPrinter() p.preamble() defer { - if sharedPrinter == nil { + if let outputDir { p.save (outputDir + "utility.swift") } } diff --git a/Generator/Generator/main.swift b/Generator/Generator/main.swift index 4408c86fa..6e44f45a9 100644 --- a/Generator/Generator/main.swift +++ b/Generator/Generator/main.swift @@ -55,10 +55,6 @@ func dropMatchingPrefix (_ enumName: String, _ enumKey: String) -> String { var globalEnums: [String: JGodotGlobalEnumElement] = [:] -let sharedPrinter: Printer? = singleFile ? Printer() : nil -var coreDefPrinter = sharedPrinter ?? Printer() -coreDefPrinter.preamble() - print ("Running with projectDir=$(projectDir) and output=\(outputDir)") let globalDocs = loadClassDoc(base: docRoot, name: "@GlobalScope") var classMap: [String:JGodotExtensionAPIClass] = [:] @@ -67,7 +63,6 @@ for x in jsonApi.classes { } var builtinMap: [String: JGodotBuiltinClass] = [:] -generateEnums(coreDefPrinter, cdef: nil, values: jsonApi.globalEnums, constantDocs: globalDocs?.constants?.constant, prefix: "") for x in jsonApi.builtinClasses { let value = x.members?.count ?? 0 > 0 @@ -87,25 +82,34 @@ for cs in jsonApi.builtinClassSizes { } } -let generatedBuiltinDir = outputDir + "/generated-builtin/" -let generatedDir = outputDir + "/generated/" +let generatedBuiltinDir: String? = singleFile ? nil : (outputDir + "/generated-builtin/") +let generatedDir: String? = singleFile ? nil : (outputDir + "/generated/") if singleFile { try! FileManager.default.createDirectory(atPath: outputDir, withIntermediateDirectories: true) -} else { +} else if let generatedBuiltinDir, let generatedDir { try! FileManager.default.createDirectory(atPath: generatedBuiltinDir, withIntermediateDirectories: true) try! FileManager.default.createDirectory(atPath: generatedDir, withIntermediateDirectories: true) } -generateBuiltinClasses(values: jsonApi.builtinClasses, outputDir: generatedBuiltinDir, sharedPrinter: sharedPrinter) -generateUtility(values: jsonApi.utilityFunctions, outputDir: generatedBuiltinDir, sharedPrinter: sharedPrinter) -generateClasses (values: jsonApi.classes, outputDir: generatedDir, sharedPrinter: sharedPrinter) - -generateCtorPointers (coreDefPrinter) -if let sharedPrinter { - sharedPrinter.save(outputDir + "/generated.swift") -} else { - coreDefPrinter.save (generatedBuiltinDir + "/core-defs.swift") +let semaphore = DispatchSemaphore(value: 0) +let _ = Task { + let coreDefPrinter = await PrinterFactory.shared.initPrinter() + coreDefPrinter.preamble() + generateEnums(coreDefPrinter, cdef: nil, values: jsonApi.globalEnums, constantDocs: globalDocs?.constants?.constant, prefix: "") + await generateBuiltinClasses(values: jsonApi.builtinClasses, outputDir: generatedBuiltinDir) + await generateUtility(values: jsonApi.utilityFunctions, outputDir: generatedBuiltinDir) + await generateClasses (values: jsonApi.classes, outputDir: generatedDir) + generateCtorPointers (coreDefPrinter) + if let generatedBuiltinDir { + coreDefPrinter.save (generatedBuiltinDir + "/core-defs.swift") + } + + if singleFile { + await PrinterFactory.shared.save(outputDir + "/generated.swift") + } + semaphore.signal() } +semaphore.wait() //print ("Done")