From 6e7397375b2256e758d3a8967a53135a0c6e641b Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Tue, 9 Apr 2019 18:06:50 +0200 Subject: [PATCH 01/10] Gather and log stats --- Sources/ResterCore/App.swift | 9 +++++ Sources/ResterCore/Stats.swift | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 Sources/ResterCore/Stats.swift diff --git a/Sources/ResterCore/App.swift b/Sources/ResterCore/App.swift index ee9e9f8..43fac19 100644 --- a/Sources/ResterCore/App.swift +++ b/Sources/ResterCore/App.swift @@ -11,6 +11,9 @@ import Path import PromiseKit +var stats = [Request.Name: Stats]() + + func before(name: Request.Name) { Current.console.display("🎬 \(name.blue) started ...\n") } @@ -21,6 +24,12 @@ func after(name: Request.Name, response: Response, result: ValidationResult) -> case .valid: let duration = format(response.elapsed).map { " (\($0)s)" } ?? "" Current.console.display("✅ \(name.blue) \("PASSED".green.bold)\(duration)\n") + stats[name, default: Stats()].add(response.elapsed) + for (name, stats) in stats.sorted(by: { $0.key > $1.key }) { + print(name.blue) + print(stats) + print() + } return true case let .invalid(message): Current.console.display(verbose: "Response:".bold) diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift new file mode 100644 index 0000000..9e2459e --- /dev/null +++ b/Sources/ResterCore/Stats.swift @@ -0,0 +1,73 @@ +// +// Stats.swift +// ResterCore +// +// Created by Sven A. Schmidt on 09/04/2019. +// + +import Foundation + + +public struct Stats { + public var durations = [TimeInterval]() + + public mutating func add(_ duration: TimeInterval) { + durations.append(duration) + } +} + + +extension Stats: CustomStringConvertible { + public var description: String { + return """ + Average: \(durations.average) + Median: \(durations.median) + Min: \(durations.min().flatMap {String($0)} ?? "-") + Max: \(durations.max().flatMap {String($0)} ?? "-") + 90% Pctl: \(durations.percentile(0.9)) + """ + } +} + + +// TODO: move to Collection+ext + +extension Collection where Element == Double { + public var average: Element { + let total = reduce(0, +) + return isEmpty ? 0 : total / Element(count) + } +} + +extension Collection where Element == Double { + public var median: Element { + let s = sorted() + if count.isMultiple(of: 2) { + return [s[count/2 - 1], s[count/2]].average + } else { + return s[count/2] + } + } +} + +extension Collection where Element == Double { + public func percentile(_ p: Double) -> Element { + let s = sorted() + let cutoff = p.clamp(max: 1.0) * Double(count) + let index = Int(cutoff) + if index == count { + return s[index - 1] + } else if Double(index) == cutoff { + return [s[index - 1], s[index]].average + } else { + return s[index] + } + } +} + + +extension Numeric where Self: Comparable { + public func clamp(max: Self) -> Self { + return min(self, max) + } +} From c7665bf2f3013501fca3320da3feb82edeeee95a Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Tue, 9 Apr 2019 18:14:14 +0200 Subject: [PATCH 02/10] Add `--stats` --- Sources/ResterCore/App.swift | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Sources/ResterCore/App.swift b/Sources/ResterCore/App.swift index 43fac19..d1e1665 100644 --- a/Sources/ResterCore/App.swift +++ b/Sources/ResterCore/App.swift @@ -11,7 +11,7 @@ import Path import PromiseKit -var stats = [Request.Name: Stats]() +var statistics: [Request.Name: Stats]? = nil func before(name: Request.Name) { @@ -24,11 +24,13 @@ func after(name: Request.Name, response: Response, result: ValidationResult) -> case .valid: let duration = format(response.elapsed).map { " (\($0)s)" } ?? "" Current.console.display("✅ \(name.blue) \("PASSED".green.bold)\(duration)\n") - stats[name, default: Stats()].add(response.elapsed) - for (name, stats) in stats.sorted(by: { $0.key > $1.key }) { - print(name.blue) - print(stats) - print() + if var stats = statistics { + stats[name, default: Stats()].add(response.elapsed) + for (name, stats) in stats.sorted(by: { $0.key > $1.key }) { + print(name.blue) + print(stats) + print() + } } return true case let .invalid(message): @@ -79,11 +81,12 @@ public let app = command( Flag("insecure", default: false, description: "do not validate SSL certificate (macOS only)"), Option("duration", default: .none, flag: "d", description: "duration to loop for"), Option("loop", default: .none, flag: "l", description: "keep executing file every seconds"), + Flag("stats", flag: "s", description: "Show stats"), Option("timeout", default: Request.defaultTimeout, flag: "t", description: "Request timeout"), Flag("verbose", flag: "v", description: "Verbose output"), Option("workdir", default: "", flag: "w", description: "Working directory (for the purpose of resolving relative paths in Restfiles)"), Argument("filename", description: "A Restfile") -) { insecure, duration, loop, timeout, verbose, workdir, filename in +) { insecure, duration, loop, stats, timeout, verbose, workdir, filename in signal(SIGINT) { s in print("\nInterrupted by user, terminating ...") @@ -97,6 +100,10 @@ public let app = command( } #endif + if stats { + statistics = [:] + } + if let loop = loop { print("Running every \(loop) seconds ...\n") var grandTotal = 0 From 3617fdd4ca46e28aa52b98295776f0b515bfcebf Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Tue, 9 Apr 2019 18:28:15 +0200 Subject: [PATCH 03/10] Improve formatting --- Sources/ResterCore/App.swift | 11 ++++------- Sources/ResterCore/Console.swift | 8 ++++++++ Sources/ResterCore/Stats.swift | 22 +++++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/Sources/ResterCore/App.swift b/Sources/ResterCore/App.swift index d1e1665..d7c1950 100644 --- a/Sources/ResterCore/App.swift +++ b/Sources/ResterCore/App.swift @@ -24,13 +24,10 @@ func after(name: Request.Name, response: Response, result: ValidationResult) -> case .valid: let duration = format(response.elapsed).map { " (\($0)s)" } ?? "" Current.console.display("✅ \(name.blue) \("PASSED".green.bold)\(duration)\n") - if var stats = statistics { - stats[name, default: Stats()].add(response.elapsed) - for (name, stats) in stats.sorted(by: { $0.key > $1.key }) { - print(name.blue) - print(stats) - print() - } + if statistics != nil { + // FIXME: avoid unsafe unwrap ("if var" will reset stats) + statistics![name, default: Stats()].add(response.elapsed) + Current.console.display(statistics!) } return true case let .invalid(message): diff --git a/Sources/ResterCore/Console.swift b/Sources/ResterCore/Console.swift index 1150906..e96adaa 100644 --- a/Sources/ResterCore/Console.swift +++ b/Sources/ResterCore/Console.swift @@ -37,6 +37,14 @@ extension Console { let failureLabel = (failed == 1) ? "failure" : "failures" display("Executed \(String(total).bold) \(testLabel), with \(failure) \(failureLabel)") } + + mutating func display(_ stats: [Request.Name: Stats]) { + for (name, stats) in stats.sorted(by: { $0.key > $1.key }) { + print(name.blue) + print(stats) + print() + } + } } diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift index 9e2459e..ab85264 100644 --- a/Sources/ResterCore/Stats.swift +++ b/Sources/ResterCore/Stats.swift @@ -20,16 +20,28 @@ public struct Stats { extension Stats: CustomStringConvertible { public var description: String { return """ - Average: \(durations.average) - Median: \(durations.median) - Min: \(durations.min().flatMap {String($0)} ?? "-") - Max: \(durations.max().flatMap {String($0)} ?? "-") - 90% Pctl: \(durations.percentile(0.9)) + Average: \(durations.average.ms) + Median: \(durations.median.ms) + Min: \(durations.min()?.ms ?? "-") + Max: \(durations.max()?.ms ?? "-") + 90% Pctl: \(durations.percentile(0.9).ms) """ } } +extension Double { + public var ms: String { + let formatter = NumberFormatter() + formatter.minimumIntegerDigits = 1 + formatter.minimumFractionDigits = 3 + formatter.maximumFractionDigits = 3 + formatter.roundingMode = .halfUp + return (formatter.string(from: NSNumber(value: self)) ?? "-") + " s" + } +} + + // TODO: move to Collection+ext extension Collection where Element == Double { From eefba0756dba2b5f0b6bcf5f30e4444b10b5e5fe Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Wed, 10 Apr 2019 08:30:05 +0200 Subject: [PATCH 04/10] Add stats tests --- Sources/ResterCore/Stats.swift | 12 ++++++--- Tests/LinuxMain.swift | 8 ++++++ Tests/ResterTests/StatsTests.swift | 39 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 Tests/ResterTests/StatsTests.swift diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift index ab85264..f3a0722 100644 --- a/Sources/ResterCore/Stats.swift +++ b/Sources/ResterCore/Stats.swift @@ -47,12 +47,13 @@ extension Double { extension Collection where Element == Double { public var average: Element { let total = reduce(0, +) - return isEmpty ? 0 : total / Element(count) + return isEmpty ? .nan : total / Element(count) } } extension Collection where Element == Double { public var median: Element { + guard !isEmpty else { return .nan } let s = sorted() if count.isMultiple(of: 2) { return [s[count/2 - 1], s[count/2]].average @@ -64,13 +65,18 @@ extension Collection where Element == Double { extension Collection where Element == Double { public func percentile(_ p: Double) -> Element { + guard count >= 2 else { return .nan } let s = sorted() - let cutoff = p.clamp(max: 1.0) * Double(count) + let cutoff = abs(p).clamp(max: 1.0) * Double(count) let index = Int(cutoff) if index == count { return s[index - 1] } else if Double(index) == cutoff { - return [s[index - 1], s[index]].average + if index == 0 { + return s[index]/2.0 + } else { + return [s[index - 1], s[index]].average + } } else { return s[index] } diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index f169a1d..763422b 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -88,6 +88,13 @@ extension RestfileDecodingTests { ("test_parse_complex_form", test_parse_complex_form), ] } +extension StatsTests { + static var allTests: [(String, (StatsTests) -> () throws -> Void)] = [ + ("test_average", test_average), + ("test_median", test_median), + ("test_percentile", test_percentile), + ] +} extension SubstitutableTests { static var allTests: [(String, (SubstitutableTests) -> () throws -> Void)] = [ ("test_substitute", test_substitute), @@ -152,6 +159,7 @@ XCTMain([ testCase(RequestTests.allTests), testCase(ResterTests.allTests), testCase(RestfileDecodingTests.allTests), + testCase(StatsTests.allTests), testCase(SubstitutableTests.allTests), testCase(TestUtilsTests.allTests), testCase(UtilsTests.allTests), diff --git a/Tests/ResterTests/StatsTests.swift b/Tests/ResterTests/StatsTests.swift new file mode 100644 index 0000000..8f8a2a3 --- /dev/null +++ b/Tests/ResterTests/StatsTests.swift @@ -0,0 +1,39 @@ +// +// StatsTests.swift +// ResterTests +// +// Created by Sven A. Schmidt on 10/04/2019. +// + +import XCTest +@testable import ResterCore + + +class StatsTests: XCTestCase { + + func test_average() { + XCTAssertEqual([1.0, 4.0, 3.0, 2.0].average, 2.5) + XCTAssertEqual([1, 4, 3, 2].average, 2.5) + XCTAssert([Double]().average.isNaN) + } + + func test_median() { + XCTAssertEqual([24, 1, 4, 5, 20, 6, 7, 12, 14, 18, 19, 22].median, 13.0) + XCTAssertEqual([1.0, 5.0, 3.0, 2.0].median, 2.5) + XCTAssertEqual([1].median, 1.0) + XCTAssert([Double]().median.isNaN) + } + + func test_percentile() { + let scores: [Double] = [43, 54, 56, 61, 62, 66, 68, 69, 69, 70, 71, 72, 77, 78, 79, 85, 87, 88, 89, 93, 95, 96, 98, 99, 99].shuffled() + XCTAssertEqual(scores.percentile(0.9), 98.0) + XCTAssertEqual(scores.percentile(1.0), 99.0) + XCTAssertEqual(scores.percentile(1.1), 99.0) + XCTAssertEqual(scores.percentile(0.5), scores.median) + XCTAssertEqual([1, 4, -3, 2, -9, -7, 0, -4, -1, 2, 1, -5, -3, 10, 10, 5].percentile(0.75), 3) + XCTAssertEqual([0, 1].percentile(0), 0) // questionable, maybe should be .nan + XCTAssert([1.0].percentile(0.5).isNaN) + XCTAssert([1.0].percentile(1.0).isNaN) + XCTAssert([1.0].percentile(0).isNaN) + } +} From b4fbb3c69f2b96b31446089f9ba59a7c4d8cabef Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Wed, 10 Apr 2019 08:47:25 +0200 Subject: [PATCH 05/10] More tests, cleaned up percentile --- Sources/ResterCore/Stats.swift | 14 +++++--------- Tests/ResterTests/StatsTests.swift | 9 +++++---- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift index f3a0722..f13bcbc 100644 --- a/Sources/ResterCore/Stats.swift +++ b/Sources/ResterCore/Stats.swift @@ -67,16 +67,12 @@ extension Collection where Element == Double { public func percentile(_ p: Double) -> Element { guard count >= 2 else { return .nan } let s = sorted() - let cutoff = abs(p).clamp(max: 1.0) * Double(count) + let cutoff = abs(p).clamp(max: 0.99) * Double(count) let index = Int(cutoff) - if index == count { - return s[index - 1] - } else if Double(index) == cutoff { - if index == 0 { - return s[index]/2.0 - } else { - return [s[index - 1], s[index]].average - } + let isInteger = (Double(index) == cutoff) + if isInteger { + guard (1.. Date: Wed, 10 Apr 2019 13:38:06 +0200 Subject: [PATCH 06/10] Stats launch test --- Sources/ResterCore/Stats.swift | 2 +- Tests/LinuxMain.swift | 1 + Tests/ResterTests/LaunchTests.swift | 7 ++++ Tests/ResterTests/TestData/basic2.yml | 13 ++++++++ Tests/ResterTests/TestUtils.swift | 6 ++-- Tests/ResterTests/TestUtilsTests.swift | 3 +- .../LaunchTests/test_launch_stats.1.txt | 32 +++++++++++++++++++ 7 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 Tests/ResterTests/TestData/basic2.yml create mode 100644 Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift index f13bcbc..a5c6d3f 100644 --- a/Sources/ResterCore/Stats.swift +++ b/Sources/ResterCore/Stats.swift @@ -37,7 +37,7 @@ extension Double { formatter.minimumFractionDigits = 3 formatter.maximumFractionDigits = 3 formatter.roundingMode = .halfUp - return (formatter.string(from: NSNumber(value: self)) ?? "-") + " s" + return (formatter.string(from: NSNumber(value: self)) ?? "-") + "s" } } diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index 763422b..93659ae 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -148,6 +148,7 @@ extension LaunchTests { ("test_launch_binary_malformed", test_launch_binary_malformed), ("test_launch_binary_loop_termination", test_launch_binary_loop_termination), ("test_launch_loop_duration", test_launch_loop_duration), + ("test_launch_stats", test_launch_stats), ] } diff --git a/Tests/ResterTests/LaunchTests.swift b/Tests/ResterTests/LaunchTests.swift index 334db75..54e91c4 100644 --- a/Tests/ResterTests/LaunchTests.swift +++ b/Tests/ResterTests/LaunchTests.swift @@ -50,4 +50,11 @@ class LaunchTests: SnapshotTestCase { XCTAssert(status == 0, "exit status not 0, was: \(status), output: \(output)") assertSnapshot(matching: output, as: .description) } + + func test_launch_stats() throws { + let requestFile = try path(fixture: "basic2.yml").unwrapped() + let (status, output) = try launch(with: requestFile, extraArguments: ["--stats"]) + XCTAssert(status == 0, "exit status not 0, was: \(status), output: \(output)") + assertSnapshot(matching: output, as: .description) + } } diff --git a/Tests/ResterTests/TestData/basic2.yml b/Tests/ResterTests/TestData/basic2.yml new file mode 100644 index 0000000..523c324 --- /dev/null +++ b/Tests/ResterTests/TestData/basic2.yml @@ -0,0 +1,13 @@ +variables: + API_URL: https://httpbin.org +requests: + request 1: + url: ${API_URL}/anything + method: GET + validation: + status: 200 + request 2: + url: ${API_URL}/anything + method: GET + validation: + status: 200 diff --git a/Tests/ResterTests/TestUtils.swift b/Tests/ResterTests/TestUtils.swift index 8f472fa..adb4f30 100644 --- a/Tests/ResterTests/TestUtils.swift +++ b/Tests/ResterTests/TestUtils.swift @@ -112,8 +112,10 @@ enum TestError: Error { extension String { var maskTime: String { - if let regex = try? Regex(pattern: "\\(\\d+\\.?\\d*s\\)") { - return regex.replaceAll(in: self, with: "(X.XXXs)") + // select 2+ decimal places to capture the timings (which are typically longers) + // and not the timeout parameter (e.g. "Request timeout: 7.0s") + if let regex = try? Regex(pattern: "\\d+\\.\\d{2,}s") { + return regex.replaceAll(in: self, with: "X.XXXs") } else { return self } diff --git a/Tests/ResterTests/TestUtilsTests.swift b/Tests/ResterTests/TestUtilsTests.swift index e867312..2999584 100644 --- a/Tests/ResterTests/TestUtilsTests.swift +++ b/Tests/ResterTests/TestUtilsTests.swift @@ -18,7 +18,8 @@ class TestUtilsTests: XCTestCase { func test_maskTime() throws { XCTAssertEqual("basic PASSED (0.01s)".maskTime, "basic PASSED (X.XXXs)") - XCTAssertEqual("basic PASSED (0s)".maskTime, "basic PASSED (X.XXXs)") + // NB: not masking timings without at least 2+ decimal places + XCTAssertEqual("basic PASSED (0s)".maskTime, "basic PASSED (0s)") } func test_maskPath() throws { diff --git a/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt b/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt new file mode 100644 index 0000000..28af083 --- /dev/null +++ b/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt @@ -0,0 +1,32 @@ +🚀 Resting basic2.yml ... + +🎬 request 1 started ... + +✅ request 1 PASSED (X.XXXs) + +request 1 +Average: X.XXXs +Median: X.XXXs +Min: X.XXXs +Max: X.XXXs +90% Pctl: NaNs + +🎬 request 2 started ... + +✅ request 2 PASSED (X.XXXs) + +request 2 +Average: X.XXXs +Median: X.XXXs +Min: X.XXXs +Max: X.XXXs +90% Pctl: NaNs + +request 1 +Average: X.XXXs +Median: X.XXXs +Min: X.XXXs +Max: X.XXXs +90% Pctl: NaNs + +Executed 2 tests, with 0 failures From 20058af7fd8da5746e9bc8fa1333f38525ffe84a Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Wed, 10 Apr 2019 14:03:06 +0200 Subject: [PATCH 07/10] Added std dev --- Sources/ResterCore/Stats.swift | 31 ++++++++++++++----- Tests/LinuxMain.swift | 1 + Tests/ResterTests/StatsTests.swift | 15 ++++++--- .../LaunchTests/test_launch_stats.1.txt | 9 ++++-- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift index a5c6d3f..2a94efd 100644 --- a/Sources/ResterCore/Stats.swift +++ b/Sources/ResterCore/Stats.swift @@ -20,30 +20,34 @@ public struct Stats { extension Stats: CustomStringConvertible { public var description: String { return """ - Average: \(durations.average.ms) - Median: \(durations.median.ms) - Min: \(durations.min()?.ms ?? "-") - Max: \(durations.max()?.ms ?? "-") - 90% Pctl: \(durations.percentile(0.9).ms) + Average: \(durations.average.fmt) + Median: \(durations.median.fmt) + Min: \(durations.min()?.fmt ?? "-") + Max: \(durations.max()?.fmt ?? "-") + Std dev: \(durations.stddev.fmt) + 90% Pctl: \(durations.percentile(0.9).fmt) """ } } extension Double { - public var ms: String { + fileprivate var fmt: String { + guard !isNaN else { return "-" } let formatter = NumberFormatter() formatter.minimumIntegerDigits = 1 formatter.minimumFractionDigits = 3 formatter.maximumFractionDigits = 3 formatter.roundingMode = .halfUp - return (formatter.string(from: NSNumber(value: self)) ?? "-") + "s" + guard let str = formatter.string(from: NSNumber(value: self)) else { return "-" } + return str + "s" } } // TODO: move to Collection+ext + extension Collection where Element == Double { public var average: Element { let total = reduce(0, +) @@ -51,6 +55,7 @@ extension Collection where Element == Double { } } + extension Collection where Element == Double { public var median: Element { guard !isEmpty else { return .nan } @@ -63,6 +68,7 @@ extension Collection where Element == Double { } } + extension Collection where Element == Double { public func percentile(_ p: Double) -> Element { guard count >= 2 else { return .nan } @@ -80,6 +86,17 @@ extension Collection where Element == Double { } +extension Collection where Element == Double { + public var stddev: Element { + guard count > 0 else { return .nan } + let mean = average + let sumOfMeanSqr = map { pow($0 - mean, 2) }.reduce(0, +) + let variance = sumOfMeanSqr / Double(count - 1) + return sqrt(variance) + } +} + + extension Numeric where Self: Comparable { public func clamp(max: Self) -> Self { return min(self, max) diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index 93659ae..31725b1 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -93,6 +93,7 @@ extension StatsTests { ("test_average", test_average), ("test_median", test_median), ("test_percentile", test_percentile), + ("test_stddev", test_stddev), ] } extension SubstitutableTests { diff --git a/Tests/ResterTests/StatsTests.swift b/Tests/ResterTests/StatsTests.swift index 000b2d9..30294ba 100644 --- a/Tests/ResterTests/StatsTests.swift +++ b/Tests/ResterTests/StatsTests.swift @@ -25,11 +25,11 @@ class StatsTests: XCTestCase { } func test_percentile() { - let scores: [Double] = [43, 54, 56, 61, 62, 66, 68, 69, 69, 70, 71, 72, 77, 78, 79, 85, 87, 88, 89, 93, 95, 96, 98, 99, 99].shuffled() - XCTAssertEqual(scores.percentile(0.9), 98.0) - XCTAssertEqual(scores.percentile(1.0), 99.0) - XCTAssertEqual(scores.percentile(1.1), 99.0) - XCTAssertEqual(scores.percentile(0.5), scores.median) + let values: [Double] = [43, 54, 56, 61, 62, 66, 68, 69, 69, 70, 71, 72, 77, 78, 79, 85, 87, 88, 89, 93, 95, 96, 98, 99, 99].shuffled() + XCTAssertEqual(values.percentile(0.9), 98.0) + XCTAssertEqual(values.percentile(1.0), 99.0) + XCTAssertEqual(values.percentile(1.1), 99.0) + XCTAssertEqual(values.percentile(0.5), values.median) XCTAssertEqual([1, 4, -3, 2, -9, -7, 0, -4, -1, 2, 1, -5, -3, 10, 10, 5].percentile(0.75), 3) XCTAssert([0, 1].percentile(0).isNaN) XCTAssert([1].percentile(0.5).isNaN) @@ -37,4 +37,9 @@ class StatsTests: XCTestCase { XCTAssert([1].percentile(0).isNaN) XCTAssert([Double]().percentile(0).isNaN) } + + func test_stddev() { + let values: [Double] = [10, 8, 10, 8, 8 , 4] + XCTAssertEqual(values.stddev, 2.19, accuracy: 0.01) + } } diff --git a/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt b/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt index 28af083..495f0cf 100644 --- a/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt +++ b/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt @@ -9,7 +9,8 @@ Average: X.XXXs Median: X.XXXs Min: X.XXXs Max: X.XXXs -90% Pctl: NaNs +Std dev: - +90% Pctl: - 🎬 request 2 started ... @@ -20,13 +21,15 @@ Average: X.XXXs Median: X.XXXs Min: X.XXXs Max: X.XXXs -90% Pctl: NaNs +Std dev: - +90% Pctl: - request 1 Average: X.XXXs Median: X.XXXs Min: X.XXXs Max: X.XXXs -90% Pctl: NaNs +Std dev: - +90% Pctl: - Executed 2 tests, with 0 failures From 89ccd9a5dc6f00bb4009cfe421349f3b4f0ca1e0 Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Wed, 10 Apr 2019 14:04:56 +0200 Subject: [PATCH 08/10] Moved extensions --- Sources/ResterCore/{ => Extensions}/Dictionary+ext.swift | 0 Sources/ResterCore/{ => Extensions}/Optional+ext.swift | 0 Sources/ResterCore/{ => Extensions}/Path+ext.swift | 0 Sources/ResterCore/{ => Extensions}/String+ext.swift | 0 Sources/ResterCore/{ => Extensions}/Value+ext.swift | 0 Tests/ResterTests/LaunchTests.swift | 1 + 6 files changed, 1 insertion(+) rename Sources/ResterCore/{ => Extensions}/Dictionary+ext.swift (100%) rename Sources/ResterCore/{ => Extensions}/Optional+ext.swift (100%) rename Sources/ResterCore/{ => Extensions}/Path+ext.swift (100%) rename Sources/ResterCore/{ => Extensions}/String+ext.swift (100%) rename Sources/ResterCore/{ => Extensions}/Value+ext.swift (100%) diff --git a/Sources/ResterCore/Dictionary+ext.swift b/Sources/ResterCore/Extensions/Dictionary+ext.swift similarity index 100% rename from Sources/ResterCore/Dictionary+ext.swift rename to Sources/ResterCore/Extensions/Dictionary+ext.swift diff --git a/Sources/ResterCore/Optional+ext.swift b/Sources/ResterCore/Extensions/Optional+ext.swift similarity index 100% rename from Sources/ResterCore/Optional+ext.swift rename to Sources/ResterCore/Extensions/Optional+ext.swift diff --git a/Sources/ResterCore/Path+ext.swift b/Sources/ResterCore/Extensions/Path+ext.swift similarity index 100% rename from Sources/ResterCore/Path+ext.swift rename to Sources/ResterCore/Extensions/Path+ext.swift diff --git a/Sources/ResterCore/String+ext.swift b/Sources/ResterCore/Extensions/String+ext.swift similarity index 100% rename from Sources/ResterCore/String+ext.swift rename to Sources/ResterCore/Extensions/String+ext.swift diff --git a/Sources/ResterCore/Value+ext.swift b/Sources/ResterCore/Extensions/Value+ext.swift similarity index 100% rename from Sources/ResterCore/Value+ext.swift rename to Sources/ResterCore/Extensions/Value+ext.swift diff --git a/Tests/ResterTests/LaunchTests.swift b/Tests/ResterTests/LaunchTests.swift index 54e91c4..a750bbc 100644 --- a/Tests/ResterTests/LaunchTests.swift +++ b/Tests/ResterTests/LaunchTests.swift @@ -57,4 +57,5 @@ class LaunchTests: SnapshotTestCase { XCTAssert(status == 0, "exit status not 0, was: \(status), output: \(output)") assertSnapshot(matching: output, as: .description) } + } From 7a083ff874501ab794b321043b6f2d07399ccd45 Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Wed, 10 Apr 2019 14:08:21 +0200 Subject: [PATCH 09/10] Move extensions to separate files --- .../Extensions/Collection+ext.swift | 57 ++++++++++++++++++ .../ResterCore/Extensions/Numeric+ext.swift | 15 +++++ Sources/ResterCore/Stats.swift | 59 ------------------- 3 files changed, 72 insertions(+), 59 deletions(-) create mode 100644 Sources/ResterCore/Extensions/Collection+ext.swift create mode 100644 Sources/ResterCore/Extensions/Numeric+ext.swift diff --git a/Sources/ResterCore/Extensions/Collection+ext.swift b/Sources/ResterCore/Extensions/Collection+ext.swift new file mode 100644 index 0000000..ea26df4 --- /dev/null +++ b/Sources/ResterCore/Extensions/Collection+ext.swift @@ -0,0 +1,57 @@ +// +// Collection+ext.swift +// ResterCore +// +// Created by Sven A. Schmidt on 10/04/2019. +// + +import Foundation + + +extension Collection where Element == Double { + public var average: Element { + let total = reduce(0, +) + return isEmpty ? .nan : total / Element(count) + } +} + + +extension Collection where Element == Double { + public var median: Element { + guard !isEmpty else { return .nan } + let s = sorted() + if count.isMultiple(of: 2) { + return [s[count/2 - 1], s[count/2]].average + } else { + return s[count/2] + } + } +} + + +extension Collection where Element == Double { + public func percentile(_ p: Double) -> Element { + guard count >= 2 else { return .nan } + let s = sorted() + let cutoff = abs(p).clamp(max: 0.99) * Double(count) + let index = Int(cutoff) + let isInteger = (Double(index) == cutoff) + if isInteger { + guard (1.. 0 else { return .nan } + let mean = average + let sumOfMeanSqr = map { pow($0 - mean, 2) }.reduce(0, +) + let variance = sumOfMeanSqr / Double(count - 1) + return sqrt(variance) + } +} diff --git a/Sources/ResterCore/Extensions/Numeric+ext.swift b/Sources/ResterCore/Extensions/Numeric+ext.swift new file mode 100644 index 0000000..75e4c89 --- /dev/null +++ b/Sources/ResterCore/Extensions/Numeric+ext.swift @@ -0,0 +1,15 @@ +// +// Numeric+ext.swift +// ResterCore +// +// Created by Sven A. Schmidt on 10/04/2019. +// + +import Foundation + + +extension Numeric where Self: Comparable { + public func clamp(max: Self) -> Self { + return min(self, max) + } +} diff --git a/Sources/ResterCore/Stats.swift b/Sources/ResterCore/Stats.swift index 2a94efd..c1c9ab9 100644 --- a/Sources/ResterCore/Stats.swift +++ b/Sources/ResterCore/Stats.swift @@ -43,62 +43,3 @@ extension Double { return str + "s" } } - - -// TODO: move to Collection+ext - - -extension Collection where Element == Double { - public var average: Element { - let total = reduce(0, +) - return isEmpty ? .nan : total / Element(count) - } -} - - -extension Collection where Element == Double { - public var median: Element { - guard !isEmpty else { return .nan } - let s = sorted() - if count.isMultiple(of: 2) { - return [s[count/2 - 1], s[count/2]].average - } else { - return s[count/2] - } - } -} - - -extension Collection where Element == Double { - public func percentile(_ p: Double) -> Element { - guard count >= 2 else { return .nan } - let s = sorted() - let cutoff = abs(p).clamp(max: 0.99) * Double(count) - let index = Int(cutoff) - let isInteger = (Double(index) == cutoff) - if isInteger { - guard (1.. 0 else { return .nan } - let mean = average - let sumOfMeanSqr = map { pow($0 - mean, 2) }.reduce(0, +) - let variance = sumOfMeanSqr / Double(count - 1) - return sqrt(variance) - } -} - - -extension Numeric where Self: Comparable { - public func clamp(max: Self) -> Self { - return min(self, max) - } -} From 5b50d3deabf8ed7ba5f2a0716e786ad9eab542ad Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Wed, 10 Apr 2019 14:26:20 +0200 Subject: [PATCH 10/10] Fix stats report sort order, cleanup --- Sources/ResterCore/App.swift | 5 ++--- Sources/ResterCore/Console.swift | 5 +++-- .../__Snapshots__/LaunchTests/test_launch_stats.1.txt | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Sources/ResterCore/App.swift b/Sources/ResterCore/App.swift index d7c1950..bbd5065 100644 --- a/Sources/ResterCore/App.swift +++ b/Sources/ResterCore/App.swift @@ -25,9 +25,8 @@ func after(name: Request.Name, response: Response, result: ValidationResult) -> let duration = format(response.elapsed).map { " (\($0)s)" } ?? "" Current.console.display("✅ \(name.blue) \("PASSED".green.bold)\(duration)\n") if statistics != nil { - // FIXME: avoid unsafe unwrap ("if var" will reset stats) - statistics![name, default: Stats()].add(response.elapsed) - Current.console.display(statistics!) + statistics?[name, default: Stats()].add(response.elapsed) + Current.console.display(statistics) } return true case let .invalid(message): diff --git a/Sources/ResterCore/Console.swift b/Sources/ResterCore/Console.swift index e96adaa..6234393 100644 --- a/Sources/ResterCore/Console.swift +++ b/Sources/ResterCore/Console.swift @@ -38,8 +38,9 @@ extension Console { display("Executed \(String(total).bold) \(testLabel), with \(failure) \(failureLabel)") } - mutating func display(_ stats: [Request.Name: Stats]) { - for (name, stats) in stats.sorted(by: { $0.key > $1.key }) { + mutating func display(_ stats: [Request.Name: Stats]?) { + guard let stats = stats else { return } + for (name, stats) in stats.sorted(by: { $0.key < $1.key }) { print(name.blue) print(stats) print() diff --git a/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt b/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt index 495f0cf..d491211 100644 --- a/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt +++ b/Tests/ResterTests/__Snapshots__/LaunchTests/test_launch_stats.1.txt @@ -16,7 +16,7 @@ Std dev: - ✅ request 2 PASSED (X.XXXs) -request 2 +request 1 Average: X.XXXs Median: X.XXXs Min: X.XXXs @@ -24,7 +24,7 @@ Max: X.XXXs Std dev: - 90% Pctl: - -request 1 +request 2 Average: X.XXXs Median: X.XXXs Min: X.XXXs