fix(minor): Fail CI also for update benchmark baseline updates that f…

…ails due to EPERM (#212)

To ensure we don't "succeed" when the benchmark throws due to permissioning errors (e.g. networking without disabling sandbox)

Plugins/BenchmarkCommandPlugin/BenchmarkCommandPlugin.swift

@@ -356,6 +356,8 @@ import PackagePlugin
                             throw MyError.benchmarkThresholdImprovement
                         case .benchmarkJobFailed:
                             print("One benchmark job failed during runtime, continuing with remaining.")
+                        case .noPermissions:
+                            throw MyError.noPermissions
                         }
                     } else {
                         print("One or more benchmarks returned an unexpected return code \(status)")
@@ -379,5 +381,6 @@ import PackagePlugin
         case baselineNotFound
         case invalidArgument
         case buildFailed
+        case noPermissions
     }
 }

Plugins/BenchmarkCommandPlugin/Command+Helpers.swift

@@ -64,4 +64,5 @@ enum ExitCode: Int32 {
     case benchmarkJobFailed = 3
     case thresholdImprovement = 4
     case baselineNotFound = 5
+    case noPermissions = 6
 }

Plugins/BenchmarkHelpGenerator/Command+Helpers.swift

@@ -64,4 +64,5 @@ enum ExitCode: Int32 {
     case benchmarkJobFailed = 3
     case thresholdImprovement = 4
     case baselineNotFound = 5
+    case noPermissions = 6
 }

Plugins/BenchmarkTool/BenchmarkTool+Baselines.swift

@@ -317,6 +317,9 @@ extension BenchmarkTool {
                 print("")
                 print("swift package --allow-writing-to-package-directory benchmark baseline update")
                 print("")
+                if let operation = baselineOperation, [.compare, .check, .update].contains(operation) {
+                    exitBenchmark(exitCode: .noPermissions)
+                }
             } else {
                 print("Failed to open file \(outputPath), errno = [\(errno)]")
             }

Plugins/BenchmarkTool/BenchmarkTool.swift

@@ -125,7 +125,8 @@ struct BenchmarkTool: AsyncParsableCommand {
         // check what failed and react accordingly
         switch exitCode {
         case .benchmarkJobFailed:
-            if baselineOperation == .check { // We need to fail with exit code for the baseline checks such that CI fails properly
+            // We need to fail with exit code for the baseline checks such that CI fails properly
+            if let operation = baselineOperation, [.compare, .check, .update].contains(operation) {
                 exitBenchmark(exitCode: .thresholdRegression)
             }
             if let failedBenchmark {
@@ -149,7 +150,8 @@ struct BenchmarkTool: AsyncParsableCommand {
         print("Likely your benchmark crashed, try running the tool in the debugger, e.g.")
         print("lldb \(benchmarkExecutablePath)")
         print("Or check Console.app for a backtrace if on macOS.")
-        if baselineOperation == .check { // We need to fail with exit code for the baseline checks such that CI fails properly
+        // We need to fail with exit code for the baseline checks such that CI fails properly
+        if let operation = baselineOperation, [.compare, .check, .update].contains(operation) {
             exitBenchmark(exitCode: .thresholdRegression)
         }
     }

Plugins/BenchmarkTool/Command+Helpers.swift

@@ -64,4 +64,5 @@ enum ExitCode: Int32 {
     case benchmarkJobFailed = 3
     case thresholdImprovement = 4
     case baselineNotFound = 5
+    case noPermissions = 6
 }