feat(minor): Enable Swift 6 mode and support specifying units per met…

…ric (#309) ## Description Swift 6 benchmark targets should now be possible with: ```swift let benchmarks: @sendable () -> Void = { ... } ``` Make it possible to specify output units for the text output for a given metric by specifying it in the configuration. E.g. ```swift Benchmark.defaultConfiguration.units = [.peakMemoryResident: .mega, .peakMemoryVirtual: .giga] ``` Also add the ability to override the time units from the command line using `--time-units`. E.g. ```bash swift package benchmark --time-units microseconds ``` This update also displays overflowing numeric values in scientific notation in the text output: ``` ╒═══════════════════════════════════════════════════════════════════════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╤═══════════╕ │ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │ ╞═══════════════════════════════════════════════════════════════════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╡ │ Samples:All metrics, full concurrency, async (ns) * │ 8.08e+07 │ 8.34e+07 │ 8.38e+07 │ 8.40e+07 │ 8.41e+07 │ 8.46e+07 │ 8.49e+07 │ 183 │ ├───────────────────────────────────────────────────────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤ │ Samples:Counter, custom metric thresholds (ns) * │ 2375 │ 2417 │ 2417 │ 2459 │ 2501 │ 2709 │ 4334 │ 4753 │ ├───────────────────────────────────────────────────────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤ │ Samples:Extended + custom metrics (ns) * │ 1875 │ 2000 │ 2000 │ 2041 │ 2167 │ 2667 │ 8750 │ 707 │ ├───────────────────────────────────────────────────────────────────────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┼───────────┤ │ Samples:Extended metrics (ns) * │ 1750 │ 1875 │ 1875 │ 1917 │ 1958 │ 2209 │ 3250 │ 705 │ ╘═══════════════════════════════════════════════════════════════════════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╧═══════════╛ ``` Also updates documentation. Addresses: #306 and #237 #293 #277 #258 --------- Co-authored-by: Axel Andersson <axel@ordo.one> Co-authored-by: dimlio <122263440+dimlio@users.noreply.github.com>
ordo-one · Feb 11, 2025 · 2bb8a39 · 2bb8a39
1 parent 51fffb6
commit 2bb8a39
Show file tree

Hide file tree

Showing 26 changed files with 354 additions and 174 deletions.
diff --git a/.spi.yml b/.spi.yml
@@ -3,4 +3,4 @@ builder:
   configs:
   - documentation_targets: [Benchmark]
     platform: linux
-    swift_version: '5.10'
+    swift_version: '6.0'
diff --git a/Package.resolved b/Package.resolved
@@ -50,8 +50,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-system",
       "state" : {
-        "revision" : "d2ba781702a1d8285419c15ee62fd734a9437ff5",
-        "version" : "1.3.2"
+        "revision" : "c8a44d836fe7913603e246acab7c528c2e780168",
+        "version" : "1.4.0"
       }
     },
     {

diff --git a/Package.swift b/Package.swift
@@ -1,4 +1,4 @@
-// swift-tools-version: 5.8
+// swift-tools-version: 5.9
 
 import class Foundation.ProcessInfo
 import PackageDescription
@@ -40,7 +40,7 @@ let package = Package(
                 )
             ),
             dependencies: [
-                "BenchmarkTool",
+                "BenchmarkTool"
             ],
             path: "Plugins/BenchmarkCommandPlugin"
         ),
@@ -50,7 +50,7 @@ let package = Package(
             name: "BenchmarkPlugin",
             capability: .buildTool(),
             dependencies: [
-                "BenchmarkBoilerplateGenerator",
+                "BenchmarkBoilerplateGenerator"
             ],
             path: "Plugins/BenchmarkPlugin"
         ),
@@ -63,6 +63,7 @@ let package = Package(
                 .product(name: "SystemPackage", package: "swift-system"),
                 .product(name: "TextTable", package: "TextTable"),
                 "Benchmark",
+                "Shared"
             ],
             path: "Plugins/BenchmarkTool"
         ),
@@ -81,7 +82,8 @@ let package = Package(
         .executableTarget(
             name: "BenchmarkHelpGenerator",
             dependencies: [
-                .product(name: "ArgumentParser", package: "swift-argument-parser")
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "Shared"
             ],
             path: "Plugins/BenchmarkHelpGenerator"
         ),
@@ -105,6 +107,9 @@ let package = Package(
         // Hooks for ARC
         .target(name: "SwiftRuntimeHooks"),
 
+        // Shared definitions
+        .target(name: "Shared"),
+
         .testTarget(
             name: "BenchmarkTests",
             dependencies: ["Benchmark"]
@@ -137,6 +142,7 @@ var dependencies: [PackageDescription.Target.Dependency] = [
     .byNameItem(name: "CLinuxOperatingSystemStats", condition: .when(platforms: [.linux])),
     .product(name: "Atomics", package: "swift-atomics"),
     "SwiftRuntimeHooks",
+    "Shared",
 ]
 
 if macOSSPIBuild == false { // jemalloc always disable for macOSSPIBuild

diff --git a/Plugins/BenchmarkCommandPlugin/BenchmarkCommandPlugin.swift b/Plugins/BenchmarkCommandPlugin/BenchmarkCommandPlugin.swift
@@ -44,6 +44,7 @@ import PackagePlugin
         let checkAbsoluteThresholds = checkAbsoluteThresholdsPath.count > 0 ? 1 : argumentExtractor.extractFlag(named: "check-absolute")
         let groupingToUse = argumentExtractor.extractOption(named: "grouping")
         let metricsToUse = argumentExtractor.extractOption(named: "metric")
+        let timeUnits = argumentExtractor.extractOption(named: "time-units")
         let debug = argumentExtractor.extractFlag(named: "debug")
         let scale = argumentExtractor.extractFlag(named: "scale")
         let helpRequested = argumentExtractor.extractFlag(named: "help")
@@ -156,6 +157,18 @@ import PackagePlugin
             args.append(contentsOf: ["--metrics", metric.description])
         }
 
+        if let firstValue = timeUnits.first {
+            if let unit = TimeUnits(rawValue: firstValue) {
+                args.append(contentsOf: ["--time-units", unit.rawValue])
+                if timeUnits.count > 1 {
+                    print("Only a single time unit may be specified, will use the first one specified '\(unit.rawValue)'")
+                }
+            } else {
+                print("Unknown time unit specified '\(firstValue)', valid units are: \(TimeUnits.allCases.map {$0.rawValue}.joined(separator: ", "))")
+                throw MyError.invalidArgument
+            }
+        }
+
         if outputFormat == .text {
             if quietRunning == 0 {
                 print("Build complete!")

diff --git a/Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift b/Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift
@@ -52,11 +52,14 @@ let help =
     --skip-target <skip-target>
                           Benchmark targets matching the regexp filter that should be skipped
     --format <format>       The output format to use, default is 'text' (values: text, markdown, influx, jmh, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds)
-    --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput, peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal,
-                          allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches, threads, threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount, retainReleaseDelta, custom)
+    --metric <metric>       Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput,
+                          peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal, allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches, threads,
+                          threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount, retainReleaseDelta, custom)
     --path <path>           The path to operate on for data export or threshold operations, default is the current directory (".") for exports and the ("./Thresholds") directory for thresholds. 
     --quiet                 Specifies that output should be suppressed (useful for if you just want to check return code)
     --scale                 Specifies that some of the text output should be scaled using the scalingFactor (denoted by '*' in output)
+    --time-units <time-units>
+                          Specifies that time related metrics output should be specified units (values: nanoseconds, microseconds, milliseconds, seconds, kiloseconds, megaseconds)
     --check-absolute        <This is deprecated, use swift package benchmark thresholds updated/check/read instead>
                           Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines.
                           This is used for CI workflows when you want to validate the thresholds vs. a persisted benchmark baseline

diff --git a/Plugins/BenchmarkCommandPlugin/Command+Helpers.swift b/Plugins/BenchmarkCommandPlugin/Command+Helpers.swift
@@ -8,12 +8,14 @@
 // http://www.apache.org/licenses/LICENSE-2.0
 //
 
-// This file need to be manually copied between the Benchmark plugin and
-// the BenchmarkTool when updated, as no external dependencies are allowed
-// for SwiftPM command tools. The source file is in BenchmarkCommandPlugin and should be
-// edited there, then manually copied to BenchmarkTool AND BenchmarkHelpGenerator.
+// This file need to be manually copied between the Benchmark plugin and the BenchmarkTool when updated,
+// as no external dependencies are allowed for SwiftPM command tools.
 
-enum Command: String, CaseIterable {
+// *************************************************************************************************************************
+// The source file is in BenchmarkCommandPlugin and should be edited there, then manually copied to Shared/Command+Helpers.
+// *************************************************************************************************************************
+
+public enum Command: String, CaseIterable {
     case run
     case list
     case baseline
@@ -23,7 +25,7 @@ enum Command: String, CaseIterable {
 }
 
 /// The benchmark data output format.
-enum OutputFormat: String, CaseIterable {
+public enum OutputFormat: String, CaseIterable {
     /// Text output formatted into a visual table suitable for console output
     case text
     /// The text output format, formatted in markdown, suitable for GitHub workflows
@@ -44,18 +46,30 @@ enum OutputFormat: String, CaseIterable {
     case metricP90AbsoluteThresholds
 }
 
-enum Grouping: String, CaseIterable {
+public enum Grouping: String, CaseIterable {
     case metric
     case benchmark
 }
 
-enum ThresholdsOperation: String, CaseIterable {
+#if swift(>=5.8)
+@_documentation(visibility: internal)
+#endif
+public enum TimeUnits: String, CaseIterable {
+    case nanoseconds
+    case microseconds
+    case milliseconds
+    case seconds
+    case kiloseconds
+    case megaseconds
+}
+
+public enum ThresholdsOperation: String, CaseIterable {
     case read
     case update
     case check
 }
 
-enum BaselineOperation: String, CaseIterable {
+public enum BaselineOperation: String, CaseIterable {
     case read
     case update
     case list
@@ -64,7 +78,7 @@ enum BaselineOperation: String, CaseIterable {
     case check
 }
 
-enum ExitCode: Int32 {
+public enum ExitCode: Int32 {
     case success = 0
     case genericFailure = 1
     case thresholdRegression = 2

diff --git a/Plugins/BenchmarkHelpGenerator/BenchmarkHelpGenerator.swift b/Plugins/BenchmarkHelpGenerator/BenchmarkHelpGenerator.swift
@@ -12,6 +12,7 @@
 // instead of writing it by hand
 
 import ArgumentParser
+import Shared
 
 let availableMetrics = [
     "cpuUser",
@@ -48,6 +49,7 @@ extension Command: ExpressibleByArgument {}
 extension Grouping: ExpressibleByArgument {}
 extension OutputFormat: ExpressibleByArgument {}
 extension BaselineOperation: ExpressibleByArgument {}
+extension TimeUnits: ExpressibleByArgument {}
 
 @main
 struct Benchmark: AsyncParsableCommand {
@@ -111,15 +113,15 @@ struct Benchmark: AsyncParsableCommand {
     @Option(name: .long, help: "The path to operate on for data export or threshold operations, default is the current directory (\".\") for exports and the (\"./Thresholds\") directory for thresholds. ")
     var path: String
 
-    @Flag(name: .long, help: "Skip building both the benchmark tool and the benchmarks to allow for faster workflows - workaround for https://github.com/swiftlang/swift-package-manager/issues/7210")
-    var skipBuild: Int
-
     @Flag(name: .long, help: "Specifies that output should be suppressed (useful for if you just want to check return code)")
     var quiet: Int
 
     @Flag(name: .long, help: "Specifies that some of the text output should be scaled using the scalingFactor (denoted by '*' in output)")
     var scale: Int
 
+    @Option(name: .long, help: "Specifies that time related metrics output should be specified units")
+    var timeUnits: TimeUnits?
+
     @Flag(name: .long, help:
         """
         <This is deprecated, use swift package benchmark thresholds updated/check/read instead>

diff --git a/Plugins/BenchmarkTool/BenchmarkTool+PrettyPrinting.swift b/Plugins/BenchmarkTool/BenchmarkTool+PrettyPrinting.swift
@@ -9,10 +9,11 @@
 //
 
 import Benchmark
+import Shared
 import SystemPackage
 import TextTable
 
-private let percentileWidth = 7
+private let percentileWidth = 9
 private let maxDescriptionWidth = 100
 
 extension OutputFormat {
@@ -37,6 +38,14 @@ extension BenchmarkTool {
         }
     }
 
+    private func formatLargeNumber(_ value: Int) -> String {
+        if abs(value) >= 10000000 { // 8 digits or more
+            let doubleValue = Double(value)
+            return String(format: "%.2e", doubleValue)
+        }
+        return "\(value)"
+    }
+
     private func formatTableEntry(_ base: Int, _ comparison: Int, _ reversePolarity: Bool = false) -> Int {
         guard comparison != 0, base != 0 else {
             return 0
@@ -89,16 +98,15 @@ extension BenchmarkTool {
                               useGroupingDescription: Bool = false) {
         let table = TextTable<ScaledResults> {
             [Column(title: title, value: "\($0.description)", width: width, align: .left),
-             Column(title: "p0", value: $0.percentiles.p0, width: percentileWidth, align: .right),
-             Column(title: "p25", value: $0.percentiles.p25, width: percentileWidth, align: .right),
-             Column(title: "p50", value: $0.percentiles.p50, width: percentileWidth, align: .right),
-             Column(title: "p75", value: $0.percentiles.p75, width: percentileWidth, align: .right),
-             Column(title: "p90", value: $0.percentiles.p90, width: percentileWidth, align: .right),
-             Column(title: "p99", value: $0.percentiles.p99, width: percentileWidth, align: .right),
-             Column(title: "p100", value: $0.percentiles.p100, width: percentileWidth, align: .right),
-             Column(title: "Samples", value: $0.samples, width: percentileWidth, align: .right)]
+             Column(title: "p0", value: formatLargeNumber($0.percentiles.p0), width: percentileWidth, align: .right),
+             Column(title: "p25", value: formatLargeNumber($0.percentiles.p25), width: percentileWidth, align: .right),
+             Column(title: "p50", value: formatLargeNumber($0.percentiles.p50), width: percentileWidth, align: .right),
+             Column(title: "p75", value: formatLargeNumber($0.percentiles.p75), width: percentileWidth, align: .right),
+             Column(title: "p90", value: formatLargeNumber($0.percentiles.p90), width: percentileWidth, align: .right),
+             Column(title: "p99", value: formatLargeNumber($0.percentiles.p99), width: percentileWidth, align: .right),
+             Column(title: "p100", value: formatLargeNumber($0.percentiles.p100), width: percentileWidth, align: .right),
+             Column(title: "Samples", value: formatLargeNumber($0.samples), width: percentileWidth, align: .right)]
         }
-
         var scaledResults: [ScaledResults] = []
         results.forEach { result in
             var resultPercentiles = ScaledResults.Percentiles()
@@ -257,14 +265,14 @@ extension BenchmarkTool {
                             let width = 40
                             let table = TextTable<ScaledResults> {
                                 [Column(title: title, value: "\($0.description)", width: width, align: .center),
-                                 Column(title: "p0", value: $0.percentiles.p0, width: percentileWidth, align: .right),
-                                 Column(title: "p25", value: $0.percentiles.p25, width: percentileWidth, align: .right),
-                                 Column(title: "p50", value: $0.percentiles.p50, width: percentileWidth, align: .right),
-                                 Column(title: "p75", value: $0.percentiles.p75, width: percentileWidth, align: .right),
-                                 Column(title: "p90", value: $0.percentiles.p90, width: percentileWidth, align: .right),
-                                 Column(title: "p99", value: $0.percentiles.p99, width: percentileWidth, align: .right),
-                                 Column(title: "p100", value: $0.percentiles.p100, width: percentileWidth, align: .right),
-                                 Column(title: "Samples", value: $0.samples, width: percentileWidth, align: .right)]
+                                 Column(title: "p0", value: formatLargeNumber($0.percentiles.p0), width: percentileWidth, align: .right),
+                                 Column(title: "p25", value: formatLargeNumber($0.percentiles.p25), width: percentileWidth, align: .right),
+                                 Column(title: "p50", value: formatLargeNumber($0.percentiles.p50), width: percentileWidth, align: .right),
+                                 Column(title: "p75", value: formatLargeNumber($0.percentiles.p75), width: percentileWidth, align: .right),
+                                 Column(title: "p90", value: formatLargeNumber($0.percentiles.p90), width: percentileWidth, align: .right),
+                                 Column(title: "p99", value: formatLargeNumber($0.percentiles.p99), width: percentileWidth, align: .right),
+                                 Column(title: "p100", value: formatLargeNumber($0.percentiles.p100), width: percentileWidth, align: .right),
+                                 Column(title: "Samples", value: formatLargeNumber($0.samples), width: percentileWidth, align: .right)]
                             }
 
                             // Rescale result to base if needed

diff --git a/Plugins/BenchmarkTool/BenchmarkTool.swift b/Plugins/BenchmarkTool/BenchmarkTool.swift
@@ -12,6 +12,7 @@
 
 import ArgumentParser
 import Benchmark
+import Shared
 import SystemPackage
 
 #if canImport(Darwin)
@@ -79,6 +80,9 @@ struct BenchmarkTool: AsyncParsableCommand {
     @Flag(name: .long, help: "True if we should scale time units, syscall rate, etc to scalingFactor")
     var scale: Bool = false
 
+    @Option(name: .long, help: "Specifies that time related metrics output should be specified units")
+    var timeUnits: TimeUnits?
+
     @Flag(name: .long, help:
         """
         Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines.
@@ -127,7 +131,7 @@ struct BenchmarkTool: AsyncParsableCommand {
         path ?? "Thresholds"
     }
 
-    mutating func failBenchmark(_ reason: String? = nil, exitCode: ExitCode = .genericFailure, _ failedBenchmark: String? = nil) {
+    mutating func failBenchmark(_ reason: String? = nil, exitCode: Shared.ExitCode = .genericFailure, _ failedBenchmark: String? = nil) {
         if let reason {
             print(reason)
             print("")
@@ -148,7 +152,7 @@ struct BenchmarkTool: AsyncParsableCommand {
         }
     }
 
-    func exitBenchmark(exitCode: ExitCode) {
+    func exitBenchmark(exitCode: Shared.ExitCode) {
         #if canImport(Darwin)
             Darwin.exit(exitCode.rawValue)
         #elseif canImport(Glibc)
@@ -354,6 +358,10 @@ struct BenchmarkTool: AsyncParsableCommand {
             args.append("--check-absolute")
         }
 
+        if let timeUnits {
+            args.append(contentsOf: ["--time-units", timeUnits.rawValue])
+        }
+
         inputFD = fromChild.readEnd.rawValue
         outputFD = toChild.writeEnd.rawValue