diff --git a/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java b/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java index e181ccb6..7ac783e1 100644 --- a/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java +++ b/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java @@ -21,6 +21,8 @@ import java.io.File; import java.math.BigDecimal; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -262,13 +264,12 @@ private static CompareState compareSingleBenchmark(ComparedBenchmark benchmarkTo } } - // TODO for the UI to call - private static Map compareAllBenchmarks(ComparisonConfig comparisonConfig, + public static Map compareAllBenchmarks(ComparisonConfig comparisonConfig, Map> benchmarksToCompare, Map>> benchmarksToCompareAgainst) { Map resultMap = new HashMap<>(); - - + resultMap = Comparisons.runComparison(comparisonConfig, benchmarksToCompare, benchmarksToCompareAgainst); + resultMap.put("timestampUTC", ZonedDateTime.now(ZoneOffset.UTC).toInstant().toString()); return resultMap; } diff --git a/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java b/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java index 927c2d3e..8b1da90d 100644 --- a/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java +++ b/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java @@ -53,9 +53,13 @@ public ComparisonConfig() { } public ComparisonConfig(Map configs) { - setScope((Scope) configs.get("scope")); - setMethod((Method) configs.get("method")); - setThreshold((Threshold) configs.get("threshold")); + String scope = (String) configs.get("scope"); + String method = (String) configs.get("method"); + String threshold = (String) configs.get("threshold"); + setScope(Scope.valueOf(scope)); + setMethod(Method.valueOf(method)); + setThreshold(Threshold.valueOf(threshold)); + setRange((String) configs.get("range")); setProjectName((String) configs.get("projectName")); setProjectVersion((String) configs.get("projectVersion")); diff --git a/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/utils/Comparisons.java b/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/utils/Comparisons.java index 6200b823..b9103cc1 100644 --- a/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/utils/Comparisons.java +++ b/gocypher-cybench-client/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/utils/Comparisons.java @@ -31,7 +31,6 @@ import com.gocypher.cybench.model.ComparedBenchmark; import com.gocypher.cybench.model.ComparedBenchmark.CompareState; import com.gocypher.cybench.model.ComparisonConfig; -import com.gocypher.cybench.model.ComparisonConfig.Scope; import com.gocypher.cybench.model.ComparisonConfig.Method; import com.gocypher.cybench.model.ComparisonConfig.Threshold; @@ -41,39 +40,96 @@ public final class Comparisons { private Comparisons() { } - public static Map runComparison(Map configMap, - Map> projectVersionBenchmarks, - Map>> compareVersionBenchmarks) { + // TODO can maybe simplify by calling runSingleComparison + public static Map runComparison(ComparisonConfig comparisonConfig, + Map> benchmarksToCompare, + Map>> benchmarksToCompareAgainst) { Map resultMap = new HashMap<>(); - - Method method = (Method) configMap.get(ConfigHandling.METHOD); - Threshold threshold = (Threshold) configMap.get(ConfigHandling.THRESHOLD); - Number deviationsAllowedNum = (Number) configMap.get(ConfigHandling.DEVIATIONS_ALLOWED); - Number percentChangeAllowedNum = (Number) configMap.get(ConfigHandling.PERCENT_CHANGE_ALLOWED); - Double deviationsAllowed = null; - Double percentChangeAllowed = null; - - if (deviationsAllowedNum != null) { - deviationsAllowed = deviationsAllowedNum.doubleValue(); - } - if (percentChangeAllowedNum != null) { - percentChangeAllowed = percentChangeAllowedNum.doubleValue(); + Method method = comparisonConfig.getMethod(); + Threshold threshold = comparisonConfig.getThreshold(); + Double deviationsAllowed = comparisonConfig.getDeviationsAllowed(); + Double percentChangeAllowed = comparisonConfig.getPercentChangeAllowed(); + + int totalComparedBenchmarks = 0; + int totalPassedBenchmarks = 0; + int totalFailedBenchmarks = 0; + int totalSkippedBenchmarks = 0; + List comparedBenchmarks = new ArrayList<>(); + List passedBenchmarks = new ArrayList<>(); + List failedBenchmarks = new ArrayList<>(); + List skippedBenchmarks = new ArrayList<>(); + + + for (Map.Entry> benchmarksToCompareEntry : benchmarksToCompare.entrySet()) { + String fingerprint = benchmarksToCompareEntry.getKey(); + Map benchmarksByMode = benchmarksToCompareEntry.getValue(); + for (Map.Entry benchmarksByModeEntry : benchmarksByMode.entrySet()) { + String mode = benchmarksByModeEntry.getKey(); + ComparedBenchmark benchmarkToCompare = benchmarksByModeEntry.getValue(); + Double score = benchmarkToCompare.getScore(); + + if (benchmarksToCompareAgainst.containsKey(fingerprint)) { + if (benchmarksToCompareAgainst.get(fingerprint).containsKey(mode)) { + List compareScores = extractScoresFromComparedBenchmarkList( + benchmarksToCompareAgainst.get(fingerprint).get(mode)); + Double compareMean = calculateMean(compareScores); + benchmarkToCompare.setCompareMean(compareMean); + + Double delta = score - compareMean; + benchmarkToCompare.setDelta(delta); + Double percentChange = calculatePercentChange(score, compareMean); + benchmarkToCompare.setPercentChange(percentChange); + Double compareSD = calculateSD(compareScores, compareMean); + benchmarkToCompare.setCompareSD(compareSD); + + Double deviationsFromMean = calculateDeviationsFromMean(score, compareMean, compareSD); + benchmarkToCompare.setDeviationsFromMean(deviationsFromMean); + + CompareState compareState = null; + if (method.equals(Method.DELTA)) { + if (threshold.equals(Threshold.GREATER)) { + compareState = passAssertionPositive(delta); + } else if (threshold.equals(Threshold.PERCENT_CHANGE)) { + compareState = passAssertionPercentage(percentChange, percentChangeAllowed); + } + } else if (method.equals(Method.SD)) { + compareState = passAssertionDeviation(deviationsFromMean, deviationsAllowed); + } + + if (compareState == CompareState.PASS) { + totalPassedBenchmarks++; + benchmarkToCompare.setCompareState(CompareState.PASS); + passedBenchmarks.add(benchmarkToCompare); + } else if (compareState == CompareState.FAIL) { + totalFailedBenchmarks++; + benchmarkToCompare.setCompareState(CompareState.FAIL); + failedBenchmarks.add(benchmarkToCompare); + } + } else { + totalSkippedBenchmarks++; + benchmarkToCompare.setCompareState(CompareState.SKIP); + skippedBenchmarks.add(benchmarkToCompare); + } + } else { + totalSkippedBenchmarks++; + benchmarkToCompare.setCompareState(CompareState.SKIP); + skippedBenchmarks.add(benchmarkToCompare); + } + + totalComparedBenchmarks++; + comparedBenchmarks.add(benchmarkToCompare); + } } - // if (method.equals(Method.DELTA)) { - // if (threshold.equals(Threshold.GREATER)) { - // resultMap = compareWithDeltaGreater(projectVersionBenchmarks, compareVersionBenchmarks, - // percentChangeAllowed); - // } else if (threshold.equals(PERCENT_CHANGE)) { - // resultMap = compareWithDeltaPercentChange(projectVersionBenchmarks, compareVersionBenchmarks, - // percentChangeAllowed); - // } - // } else if (method.equals(SD)) { - // resultMap = compareWithSD(projectVersionBenchmarks, compareVersionBenchmarks, deviationsAllowed); - // } - - // resultMap.put("timestampUTC", ZonedDateTime.now(ZoneOffset.UTC).toInstant().toString()); + resultMap.put("totalComparedBenchmarks", totalComparedBenchmarks); + resultMap.put("totalPassedBenchmarks", totalPassedBenchmarks); + resultMap.put("totalFailedBenchmarks", totalFailedBenchmarks); + resultMap.put("totalSkippedBenchmarks", totalSkippedBenchmarks); + resultMap.put("comparedBenchmarks", comparedBenchmarks); + resultMap.put("passedBenchmarks", passedBenchmarks); + resultMap.put("failedBenchmarks", failedBenchmarks); + resultMap.put("skippedBenchmarks", skippedBenchmarks); return resultMap; }