Skip to content

Commit

Permalink
add functionality for UI application to run comparisons
Browse files Browse the repository at this point in the history
  • Loading branch information
@spencerspitz
spencerspitz committed Feb 7, 2022
1 parent 2158437 commit feef417
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 37 deletions.
9 changes: 5 additions & 4 deletions ...ent/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@

import java.io.File;
import java.math.BigDecimal;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -262,13 +264,12 @@ private static CompareState compareSingleBenchmark(ComparedBenchmark benchmarkTo
}
}

// TODO for the UI to call
private static Map<String, Object> compareAllBenchmarks(ComparisonConfig comparisonConfig,
public static Map<String, Object> compareAllBenchmarks(ComparisonConfig comparisonConfig,
Map<String, Map<String, ComparedBenchmark>> benchmarksToCompare,
Map<String, Map<String, List<ComparedBenchmark>>> benchmarksToCompareAgainst) {
Map<String, Object> resultMap = new HashMap<>();


resultMap = Comparisons.runComparison(comparisonConfig, benchmarksToCompare, benchmarksToCompareAgainst);
resultMap.put("timestampUTC", ZonedDateTime.now(ZoneOffset.UTC).toInstant().toString());
return resultMap;
}

Expand Down
10 changes: 7 additions & 3 deletions ...ocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,13 @@ public ComparisonConfig() {
}

public ComparisonConfig(Map<String, Object> configs) {
setScope((Scope) configs.get("scope"));
setMethod((Method) configs.get("method"));
setThreshold((Threshold) configs.get("threshold"));
String scope = (String) configs.get("scope");
String method = (String) configs.get("method");
String threshold = (String) configs.get("threshold");
setScope(Scope.valueOf(scope));
setMethod(Method.valueOf(method));
setThreshold(Threshold.valueOf(threshold));

setRange((String) configs.get("range"));
setProjectName((String) configs.get("projectName"));
setProjectVersion((String) configs.get("projectVersion"));
Expand Down
116 changes: 86 additions & 30 deletions ...ent/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/utils/Comparisons.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import com.gocypher.cybench.model.ComparedBenchmark;
import com.gocypher.cybench.model.ComparedBenchmark.CompareState;
import com.gocypher.cybench.model.ComparisonConfig;
import com.gocypher.cybench.model.ComparisonConfig.Scope;
import com.gocypher.cybench.model.ComparisonConfig.Method;
import com.gocypher.cybench.model.ComparisonConfig.Threshold;

Expand All @@ -41,39 +40,96 @@ public final class Comparisons {
private Comparisons() {
}

public static Map<String, Object> runComparison(Map<String, Object> configMap,
Map<String, Map<String, ComparedBenchmark>> projectVersionBenchmarks,
Map<String, Map<String, List<ComparedBenchmark>>> compareVersionBenchmarks) {
// TODO can maybe simplify by calling runSingleComparison
public static Map<String, Object> runComparison(ComparisonConfig comparisonConfig,
Map<String, Map<String, ComparedBenchmark>> benchmarksToCompare,
Map<String, Map<String, List<ComparedBenchmark>>> benchmarksToCompareAgainst) {
Map<String, Object> resultMap = new HashMap<>();


Method method = (Method) configMap.get(ConfigHandling.METHOD);
Threshold threshold = (Threshold) configMap.get(ConfigHandling.THRESHOLD);
Number deviationsAllowedNum = (Number) configMap.get(ConfigHandling.DEVIATIONS_ALLOWED);
Number percentChangeAllowedNum = (Number) configMap.get(ConfigHandling.PERCENT_CHANGE_ALLOWED);
Double deviationsAllowed = null;
Double percentChangeAllowed = null;

if (deviationsAllowedNum != null) {
deviationsAllowed = deviationsAllowedNum.doubleValue();
}
if (percentChangeAllowedNum != null) {
percentChangeAllowed = percentChangeAllowedNum.doubleValue();
Method method = comparisonConfig.getMethod();
Threshold threshold = comparisonConfig.getThreshold();
Double deviationsAllowed = comparisonConfig.getDeviationsAllowed();
Double percentChangeAllowed = comparisonConfig.getPercentChangeAllowed();

int totalComparedBenchmarks = 0;
int totalPassedBenchmarks = 0;
int totalFailedBenchmarks = 0;
int totalSkippedBenchmarks = 0;
List<ComparedBenchmark> comparedBenchmarks = new ArrayList<>();
List<ComparedBenchmark> passedBenchmarks = new ArrayList<>();
List<ComparedBenchmark> failedBenchmarks = new ArrayList<>();
List<ComparedBenchmark> skippedBenchmarks = new ArrayList<>();


for (Map.Entry<String, Map<String, ComparedBenchmark>> benchmarksToCompareEntry : benchmarksToCompare.entrySet()) {
String fingerprint = benchmarksToCompareEntry.getKey();
Map<String, ComparedBenchmark> benchmarksByMode = benchmarksToCompareEntry.getValue();
for (Map.Entry<String, ComparedBenchmark> benchmarksByModeEntry : benchmarksByMode.entrySet()) {
String mode = benchmarksByModeEntry.getKey();
ComparedBenchmark benchmarkToCompare = benchmarksByModeEntry.getValue();
Double score = benchmarkToCompare.getScore();

if (benchmarksToCompareAgainst.containsKey(fingerprint)) {
if (benchmarksToCompareAgainst.get(fingerprint).containsKey(mode)) {
List<Double> compareScores = extractScoresFromComparedBenchmarkList(
benchmarksToCompareAgainst.get(fingerprint).get(mode));
Double compareMean = calculateMean(compareScores);
benchmarkToCompare.setCompareMean(compareMean);

Double delta = score - compareMean;
benchmarkToCompare.setDelta(delta);
Double percentChange = calculatePercentChange(score, compareMean);
benchmarkToCompare.setPercentChange(percentChange);
Double compareSD = calculateSD(compareScores, compareMean);
benchmarkToCompare.setCompareSD(compareSD);

Double deviationsFromMean = calculateDeviationsFromMean(score, compareMean, compareSD);
benchmarkToCompare.setDeviationsFromMean(deviationsFromMean);

CompareState compareState = null;
if (method.equals(Method.DELTA)) {
if (threshold.equals(Threshold.GREATER)) {
compareState = passAssertionPositive(delta);
} else if (threshold.equals(Threshold.PERCENT_CHANGE)) {
compareState = passAssertionPercentage(percentChange, percentChangeAllowed);
}
} else if (method.equals(Method.SD)) {
compareState = passAssertionDeviation(deviationsFromMean, deviationsAllowed);
}

if (compareState == CompareState.PASS) {
totalPassedBenchmarks++;
benchmarkToCompare.setCompareState(CompareState.PASS);
passedBenchmarks.add(benchmarkToCompare);
} else if (compareState == CompareState.FAIL) {
totalFailedBenchmarks++;
benchmarkToCompare.setCompareState(CompareState.FAIL);
failedBenchmarks.add(benchmarkToCompare);
}
} else {
totalSkippedBenchmarks++;
benchmarkToCompare.setCompareState(CompareState.SKIP);
skippedBenchmarks.add(benchmarkToCompare);
}
} else {
totalSkippedBenchmarks++;
benchmarkToCompare.setCompareState(CompareState.SKIP);
skippedBenchmarks.add(benchmarkToCompare);
}

totalComparedBenchmarks++;
comparedBenchmarks.add(benchmarkToCompare);
}
}

// if (method.equals(Method.DELTA)) {
// if (threshold.equals(Threshold.GREATER)) {
// resultMap = compareWithDeltaGreater(projectVersionBenchmarks, compareVersionBenchmarks,
// percentChangeAllowed);
// } else if (threshold.equals(PERCENT_CHANGE)) {
// resultMap = compareWithDeltaPercentChange(projectVersionBenchmarks, compareVersionBenchmarks,
// percentChangeAllowed);
// }
// } else if (method.equals(SD)) {
// resultMap = compareWithSD(projectVersionBenchmarks, compareVersionBenchmarks, deviationsAllowed);
// }

// resultMap.put("timestampUTC", ZonedDateTime.now(ZoneOffset.UTC).toInstant().toString());
resultMap.put("totalComparedBenchmarks", totalComparedBenchmarks);
resultMap.put("totalPassedBenchmarks", totalPassedBenchmarks);
resultMap.put("totalFailedBenchmarks", totalFailedBenchmarks);
resultMap.put("totalSkippedBenchmarks", totalSkippedBenchmarks);
resultMap.put("comparedBenchmarks", comparedBenchmarks);
resultMap.put("passedBenchmarks", passedBenchmarks);
resultMap.put("failedBenchmarks", failedBenchmarks);
resultMap.put("skippedBenchmarks", skippedBenchmarks);
return resultMap;
}

Expand Down

0 comments on commit feef417

Please sign in to comment.