ensure a report isn't compared against itself

Nastel · Feb 7, 2022 · bf8e0a6 · bf8e0a6
1 parent feef417
commit bf8e0a6
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 16 deletions.
diff --git a/...ent/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java b/...ent/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/CompareBenchmarks.java
@@ -24,6 +24,7 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -224,13 +225,13 @@ private static CompareState compareSingleBenchmark(ComparedBenchmark benchmarkTo
 
             if (Requests.allProjectVersions.contains(compareVersion)) {
                 int range;
-                int maxRange = Requests.allProjectReportSummaries.get(compareVersion).size();
+                int maxRange = Requests.reportSummaries.get(compareVersion).size();
                 if (compareRange.equals("ALL")) {
                     range = maxRange;
                 } else {
                     range = Integer.parseInt(compareRange);
                     if (range > maxRange) {
-                        logWarn("SKIP COMPARISON - {} : mode={} - There are not enough values to compare to in version={} with specific range={}",
+                        logWarn("SKIP COMPARISON - {} : mode={} - There are not enough reports to compare to in version={} with specific range={}",
                                 benchmarkToCompare.getDisplayName(), benchmarkToCompare.getMode(), Requests.currentVersion, range);
                         return CompareState.SKIP;
                     }
@@ -240,16 +241,27 @@ private static CompareState compareSingleBenchmark(ComparedBenchmark benchmarkTo
 
 
                 List<ComparedBenchmark> benchmarksToCompareAgainst = new ArrayList<>();
-                for (int i=0; i < range; i++) {
-                    String reportID = (String) Requests.allProjectReportSummaries.get(compareVersion).get(i).get("reportId");
-                    if (!Requests.fetchedBenchmarks.containsKey(reportID)) {
-                        Requests.getInstance().getBenchmarksInReport(reportID, accessToken);
-                    }
-                    Map<String, List<ComparedBenchmark>> benchmaksInReportWithSameFingerprint = Requests.fetchedBenchmarks.get(reportID).get(benchmarkFingerprint);
-                    if (benchmaksInReportWithSameFingerprint.containsKey(benchmarkToCompare.getMode())) {
-                        benchmarksToCompareAgainst.addAll(benchmaksInReportWithSameFingerprint.get(benchmarkToCompare.getMode()));
+                int added = 0;
+                for (int i=0; i < Requests.reportSummaries.get(compareVersion).size() && added < range; i++) {
+                    String reportID = (String) Requests.reportSummaries.get(compareVersion).get(i).get("reportId");
+                    Number timestampNum = (Number) Requests.reportSummaries.get(compareVersion).get(i).get("timestamp");
+                    Date reportDateTime = new Date(timestampNum.longValue() / 1000);
+                    if (reportID != Requests.recentReportID && Requests.recentReportDateTime.compareTo(reportDateTime) != 0) {
+                        if (!Requests.fetchedBenchmarks.containsKey(reportID)) {
+                            Requests.getInstance().getBenchmarksInReport(reportID, accessToken);
+                        }
+                        Map<String, List<ComparedBenchmark>> benchmaksInReportWithSameFingerprint = Requests.fetchedBenchmarks.get(reportID).get(benchmarkFingerprint);
+                        if (benchmaksInReportWithSameFingerprint.containsKey(benchmarkToCompare.getMode())) {
+                            benchmarksToCompareAgainst.addAll(benchmaksInReportWithSameFingerprint.get(benchmarkToCompare.getMode()));
+                            added++;
+                        }
                     }
-
+                }
+
+                if (benchmarksToCompareAgainst.size() == 0) {
+                    logWarn("SKIP COMPARISON - {} : mode={} - There are not enough benchmarks to compare to in version={} with specific range={}",
+                                benchmarkToCompare.getDisplayName(), benchmarkToCompare.getMode(), Requests.currentVersion, range);
+                        return CompareState.SKIP;
                 }
 
                 return Comparisons.runSingleComparison(benchmarkToCompare, benchmarksToCompareAgainst);

diff --git a/...ocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java b/...ocypher-cybench-comparator/src/main/java/com/gocypher/cybench/model/ComparisonConfig.java
@@ -53,9 +53,9 @@ public ComparisonConfig() {
     }
 
     public ComparisonConfig(Map<String, Object> configs) {
-        String scope = (String) configs.get("scope");
-        String method = (String) configs.get("method");
-        String threshold = (String) configs.get("threshold");
+        String scope = configs.get("scope").toString();
+        String method = configs.get("method").toString();
+        String threshold = configs.get("threshold").toString();
         setScope(Scope.valueOf(scope));
         setMethod(Method.valueOf(method));
         setThreshold(Threshold.valueOf(threshold));

diff --git a/...ent/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/services/Requests.java b/...ent/gocypher-cybench-comparator/src/main/java/com/gocypher/cybench/services/Requests.java
@@ -53,9 +53,14 @@ public class Requests {
     public static String currentVersion = null;
     public static String latestVersion = null;
     public static String previousVersion = null;
+
+    // recent report identifiers
+    public static String recentReportID = null;
+    public static Date recentReportDateTime = null;
+
     public static ArrayList<String> allProjectVersions = new ArrayList<>();
     // version: list of reports
-    public static Map<String, ArrayList<Map<String, Object>>> allProjectReportSummaries = new HashMap<>();
+    public static Map<String, ArrayList<Map<String, Object>>> reportSummaries = new HashMap<>();
     // <ReportID: <Benchmark Fingerprint : <Mode : List<ComparedBenchmark>>>>>
     public static Map<String, Map<String, Map<String, List<ComparedBenchmark>>>> fetchedBenchmarks = new HashMap<>();
 
@@ -100,6 +105,9 @@ public static Map<String, Map<String, ComparedBenchmark>> parseRecentReport(File
                 String[] parsedURL = reportURL.split(benchmarkBaseUrl);
                 reportID = parsedURL[1].split("/")[0];
             }
+            recentReportID = reportID;
+            Number timestampNum = (Number) report.get("timestamp");
+            recentReportDateTime = new Date(timestampNum.longValue());
             project = (String) report.get("project");
             currentVersion = (String) report.get("projectVersion");
             JSONObject categories = (JSONObject) report.get("benchmarks");
@@ -154,7 +162,7 @@ public boolean getProjectSummary(String projectName, String accessToken) {
                 latestVersion = (String) projectSummary.get("latestVersion");
                 previousVersion = (String) projectSummary.get("previousVersion");
                 allProjectVersions = (ArrayList<String>) projectSummary.get("versions");
-                allProjectReportSummaries = (Map<String, ArrayList<Map<String, Object>>>) projectSummary.get("reports");
+                reportSummaries = (Map<String, ArrayList<Map<String, Object>>>) projectSummary.get("reports");
             } catch (Exception e) {
                 log.error("* Failed to fetch project data for " + projectName, e);
                 return false;