Merge pull request #56 from finestructure/develop

release-0.5.0

Sources/ResterCore/App.swift

@@ -11,6 +11,9 @@ import Path
 import PromiseKit
 
 
+var statistics: [Request.Name: Stats]? = nil
+
+
 func before(name: Request.Name) {
     Current.console.display("🎬  \(name.blue) started ...\n")
 }
@@ -21,6 +24,10 @@ func after(name: Request.Name, response: Response, result: ValidationResult) ->
     case .valid:
         let duration = format(response.elapsed).map { " (\($0)s)" } ?? ""
         Current.console.display("✅  \(name.blue) \("PASSED".green.bold)\(duration)\n")
+        if statistics != nil {
+            statistics?[name, default: Stats()].add(response.elapsed)
+            Current.console.display(statistics)
+        }
         return true
     case let .invalid(message):
         Current.console.display(verbose: "Response:".bold)
@@ -70,11 +77,12 @@ public let app = command(
     Flag("insecure", default: false, description: "do not validate SSL certificate (macOS only)"),
     Option<Int?>("duration", default: .none, flag: "d", description: "duration <seconds> to loop for"),
     Option<Int?>("loop", default: .none, flag: "l", description: "keep executing file every <loop> seconds"),
+    Flag("stats", flag: "s", description: "Show stats"),
     Option<TimeInterval>("timeout", default: Request.defaultTimeout, flag: "t", description: "Request timeout"),
     Flag("verbose", flag: "v", description: "Verbose output"),
     Option<String>("workdir", default: "", flag: "w", description: "Working directory (for the purpose of resolving relative paths in Restfiles)"),
     Argument<String>("filename", description: "A Restfile")
-) { insecure, duration, loop, timeout, verbose, workdir, filename in
+) { insecure, duration, loop, stats, timeout, verbose, workdir, filename in
 
     signal(SIGINT) { s in
         print("\nInterrupted by user, terminating ...")
@@ -88,6 +96,10 @@ public let app = command(
     }
     #endif
 
+    if stats {
+        statistics = [:]
+    }
+
     if let loop = loop {
         print("Running every \(loop) seconds ...\n")
         var grandTotal = 0

Sources/ResterCore/Console.swift

@@ -37,6 +37,15 @@ extension Console {
         let failureLabel = (failed == 1) ? "failure" : "failures"
         display("Executed \(String(total).bold) \(testLabel), with \(failure) \(failureLabel)")
     }
+
+    mutating func display(_ stats: [Request.Name: Stats]?) {
+        guard let stats = stats else { return }
+        for (name, stats) in stats.sorted(by: { $0.key < $1.key }) {
+            print(name.blue)
+            print(stats)
+            print()
+        }
+    }
 }
 
 

Sources/ResterCore/Extensions/Collection+ext.swift

@@ -0,0 +1,57 @@
+//
+//  Collection+ext.swift
+//  ResterCore
+//
+//  Created by Sven A. Schmidt on 10/04/2019.
+//
+
+import Foundation
+
+
+extension Collection where Element == Double {
+    public var average: Element {
+        let total = reduce(0, +)
+        return isEmpty ? .nan : total / Element(count)
+    }
+}
+
+
+extension Collection where Element == Double {
+    public var median: Element {
+        guard !isEmpty else { return .nan }
+        let s = sorted()
+        if count.isMultiple(of: 2) {
+            return [s[count/2 - 1], s[count/2]].average
+        } else {
+            return s[count/2]
+        }
+    }
+}
+
+
+extension Collection where Element == Double {
+    public func percentile(_ p: Double) -> Element {
+        guard count >= 2 else { return .nan }
+        let s = sorted()
+        let cutoff = abs(p).clamp(max: 0.99) * Double(count)
+        let index = Int(cutoff)
+        let isInteger = (Double(index) == cutoff)
+        if isInteger {
+            guard (1..<count).contains(index) else { return .nan }
+            return [s[index - 1], s[index]].average
+        } else {
+            return s[index]
+        }
+    }
+}
+
+
+extension Collection where Element == Double {
+    public var stddev: Element {
+        guard count > 0 else { return .nan }
+        let mean = average
+        let sumOfMeanSqr = map { pow($0 - mean, 2) }.reduce(0, +)
+        let variance = sumOfMeanSqr / Double(count - 1)
+        return sqrt(variance)
+    }
+}

Sources/ResterCore/Extensions/Numeric+ext.swift

@@ -0,0 +1,15 @@
+//
+//  Numeric+ext.swift
+//  ResterCore
+//
+//  Created by Sven A. Schmidt on 10/04/2019.
+//
+
+import Foundation
+
+
+extension Numeric where Self: Comparable {
+    public func clamp(max: Self) -> Self {
+        return min(self, max)
+    }
+}

Sources/ResterCore/Stats.swift

@@ -0,0 +1,45 @@
+//
+//  Stats.swift
+//  ResterCore
+//
+//  Created by Sven A. Schmidt on 09/04/2019.
+//
+
+import Foundation
+
+
+public struct Stats {
+    public var durations = [TimeInterval]()
+
+    public mutating func add(_ duration: TimeInterval) {
+        durations.append(duration)
+    }
+}
+
+
+extension Stats: CustomStringConvertible {
+    public var description: String {
+        return """
+        Average:   \(durations.average.fmt)
+        Median:    \(durations.median.fmt)
+        Min:       \(durations.min()?.fmt ?? "-")
+        Max:       \(durations.max()?.fmt ?? "-")
+        Std dev:   \(durations.stddev.fmt)
+        90% Pctl:  \(durations.percentile(0.9).fmt)
+        """
+    }
+}
+
+
+extension Double {
+    fileprivate var fmt: String {
+        guard !isNaN else { return "-" }
+        let formatter = NumberFormatter()
+        formatter.minimumIntegerDigits = 1
+        formatter.minimumFractionDigits = 3
+        formatter.maximumFractionDigits = 3
+        formatter.roundingMode = .halfUp
+        guard let str = formatter.string(from: NSNumber(value: self)) else { return "-" }
+        return str + "s"
+    }
+}

Tests/LinuxMain.swift

@@ -88,6 +88,14 @@ extension RestfileDecodingTests {
       ("test_parse_complex_form", test_parse_complex_form),
   ]
 }
+extension StatsTests {
+  static var allTests: [(String, (StatsTests) -> () throws -> Void)] = [
+      ("test_average", test_average),
+      ("test_median", test_median),
+      ("test_percentile", test_percentile),
+      ("test_stddev", test_stddev),
+  ]
+}
 extension SubstitutableTests {
   static var allTests: [(String, (SubstitutableTests) -> () throws -> Void)] = [
       ("test_substitute", test_substitute),
@@ -141,6 +149,7 @@ extension LaunchTests {
       ("test_launch_binary_malformed", test_launch_binary_malformed),
       ("test_launch_binary_loop_termination", test_launch_binary_loop_termination),
       ("test_launch_loop_duration", test_launch_loop_duration),
+      ("test_launch_stats", test_launch_stats),
   ]
 }
 
@@ -152,6 +161,7 @@ XCTMain([
   testCase(RequestTests.allTests),
   testCase(ResterTests.allTests),
   testCase(RestfileDecodingTests.allTests),
+  testCase(StatsTests.allTests),
   testCase(SubstitutableTests.allTests),
   testCase(TestUtilsTests.allTests),
   testCase(UtilsTests.allTests),

Tests/ResterTests/LaunchTests.swift

@@ -50,4 +50,12 @@ class LaunchTests: SnapshotTestCase {
         XCTAssert(status == 0, "exit status not 0, was: \(status), output: \(output)")
         assertSnapshot(matching: output, as: .description)
     }
+
+    func test_launch_stats() throws {
+        let requestFile = try path(fixture: "basic2.yml").unwrapped()
+        let (status, output) = try launch(with: requestFile, extraArguments: ["--stats"])
+        XCTAssert(status == 0, "exit status not 0, was: \(status), output: \(output)")
+        assertSnapshot(matching: output, as: .description)
+    }
+
 }

Tests/ResterTests/StatsTests.swift

@@ -0,0 +1,45 @@
+//
+//  StatsTests.swift
+//  ResterTests
+//
+//  Created by Sven A. Schmidt on 10/04/2019.
+//
+
+import XCTest
+@testable import ResterCore
+
+
+class StatsTests: XCTestCase {
+
+    func test_average() {
+        XCTAssertEqual([1.0, 4.0, 3.0, 2.0].average, 2.5)
+        XCTAssertEqual([1, 4, 3, 2].average, 2.5)
+        XCTAssert([Double]().average.isNaN)
+    }
+
+    func test_median() {
+        XCTAssertEqual([24, 1, 4, 5, 20, 6, 7, 12, 14, 18, 19, 22].median, 13.0)
+        XCTAssertEqual([1.0, 5.0, 3.0, 2.0].median, 2.5)
+        XCTAssertEqual([1].median, 1.0)
+        XCTAssert([Double]().median.isNaN)
+    }
+
+    func test_percentile() {
+        let values: [Double] = [43, 54, 56, 61, 62, 66, 68, 69, 69, 70, 71, 72, 77, 78, 79, 85, 87, 88, 89, 93, 95, 96, 98, 99, 99].shuffled()
+        XCTAssertEqual(values.percentile(0.9), 98.0)
+        XCTAssertEqual(values.percentile(1.0), 99.0)
+        XCTAssertEqual(values.percentile(1.1), 99.0)
+        XCTAssertEqual(values.percentile(0.5), values.median)
+        XCTAssertEqual([1, 4, -3, 2, -9, -7, 0, -4, -1, 2, 1, -5, -3, 10, 10, 5].percentile(0.75), 3)
+        XCTAssert([0, 1].percentile(0).isNaN)
+        XCTAssert([1].percentile(0.5).isNaN)
+        XCTAssert([1].percentile(1.0).isNaN)
+        XCTAssert([1].percentile(0).isNaN)
+        XCTAssert([Double]().percentile(0).isNaN)
+    }
+
+    func test_stddev() {
+        let values: [Double] = [10, 8, 10, 8, 8 , 4]
+        XCTAssertEqual(values.stddev, 2.19, accuracy: 0.01)
+    }
+}

Tests/ResterTests/TestData/basic2.yml

@@ -0,0 +1,13 @@
+variables:
+  API_URL: https://httpbin.org
+requests:
+  request 1:
+    url: ${API_URL}/anything
+    method: GET
+    validation:
+      status: 200
+  request 2:
+    url: ${API_URL}/anything
+    method: GET
+    validation:
+      status: 200

Tests/ResterTests/TestUtils.swift

@@ -112,8 +112,10 @@ enum TestError: Error {
 
 extension String {
     var maskTime: String {
-        if let regex = try? Regex(pattern: "\\(\\d+\\.?\\d*s\\)") {
-            return regex.replaceAll(in: self, with: "(X.XXXs)")
+        // select 2+ decimal places to capture the timings (which are typically longers)
+        // and not the timeout parameter (e.g. "Request timeout: 7.0s")
+        if let regex = try? Regex(pattern: "\\d+\\.\\d{2,}s") {
+            return regex.replaceAll(in: self, with: "X.XXXs")
         } else {
             return self
         }

Tests/ResterTests/TestUtilsTests.swift

@@ -18,7 +18,8 @@ class TestUtilsTests: XCTestCase {
 
     func test_maskTime() throws {
         XCTAssertEqual("basic PASSED (0.01s)".maskTime, "basic PASSED (X.XXXs)")
-        XCTAssertEqual("basic PASSED (0s)".maskTime, "basic PASSED (X.XXXs)")
+        // NB: not masking timings without at least 2+ decimal places
+        XCTAssertEqual("basic PASSED (0s)".maskTime, "basic PASSED (0s)")
     }
 
     func test_maskPath() throws {

Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt

@@ -0,0 +1,35 @@
+🚀  Resting basic2.yml ...
+
+🎬  request 1 started ...
+
+✅  request 1 PASSED (X.XXXs)
+
+request 1
+Average:   X.XXXs
+Median:    X.XXXs
+Min:       X.XXXs
+Max:       X.XXXs
+Std dev:   -
+90% Pctl:  -
+
+🎬  request 2 started ...
+
+✅  request 2 PASSED (X.XXXs)
+
+request 1
+Average:   X.XXXs
+Median:    X.XXXs
+Min:       X.XXXs
+Max:       X.XXXs
+Std dev:   -
+90% Pctl:  -
+
+request 2
+Average:   X.XXXs
+Median:    X.XXXs
+Min:       X.XXXs
+Max:       X.XXXs
+Std dev:   -
+90% Pctl:  -
+
+Executed 2 tests, with 0 failures