Support reporting geometric mean by benchmark tags (#132)

This reports geometric mean organized by the tag(s) assigned to each benchmark.
This will allow us to include benchmarks in the pyperformance suite that we
don't necessarily want to include in "one big overall number" to represent progress.

Addresses python/pyperformance#208

Author: mdboom

doc/api.rst

@@ -676,6 +676,7 @@ Benchmark:
 * ``inner_loops`` (``int >= 1``): number of inner-loops of the benchmark (``int``)
 * ``timer``: Implementation of ``time.perf_counter()``, and also resolution if
   available
+* ``tags``: (list of str, optional): A list of tags associated with the benchmark. If provided, the results output will be aggreggated by each tag.
 
 Python metadata:
 
@@ -831,6 +832,7 @@ Example of JSON, ``...`` is used in the example for readability::
             "loops": 8,
             "name": "telco",
             "perf_version": "0.8.2",
+            "tags": ["numeric"],
             ...
         },
         "version": "1.0"

pyperf/_compare.py

@@ -49,6 +49,10 @@ def format_geometric_mean(norm_means):
     return format_normalized_mean(geo_mean)
 
 
+def get_tags_for_result(result):
+    return result.ref.benchmark.get_metadata().get("tags", [])
+
+
 class CompareResult(object):
     def __init__(self, ref, changed, min_speed=None):
         # CompareData object
@@ -242,6 +246,12 @@ def __init__(self, benchmarks, args):
 
         self.show_name = (len(grouped_by_name) > 1)
 
+        self.tags = set()
+        for results in self.all_results:
+            for result in results:
+                self.tags.update(get_tags_for_result(result))
+        self.tags = sorted(list(self.tags))
+
     def compare_benchmarks(self, name, benchmarks):
         min_speed = self.min_speed
 
@@ -258,11 +268,11 @@ def compare_benchmarks(self, name, benchmarks):
         return results
 
     @staticmethod
-    def display_not_signiticant(not_significant):
+    def display_not_significant(not_significant):
         print("Benchmark hidden because not significant (%s): %s"
               % (len(not_significant), ', '.join(not_significant)))
 
-    def compare_suites_table(self):
+    def compare_suites_table(self, all_results):
         if self.group_by_speed:
             def sort_key(results):
                 result = results[0]
@@ -280,7 +290,7 @@ def sort_key(results):
 
         rows = []
         not_significant = []
-        for results in self.all_results:
+        for results in all_results:
             row = [results.name]
 
             ref_bench = results[0].ref.benchmark
@@ -324,14 +334,14 @@ def sort_key(results):
         if not_significant:
             if rows:
                 print()
-            self.display_not_signiticant(not_significant)
+            self.display_not_significant(not_significant)
 
-    def compare_suites_by_speed(self):
+    def compare_suites_by_speed(self, all_results):
         not_significant = []
         slower = []
         faster = []
         same = []
-        for results in self.all_results:
+        for results in all_results:
             result = results[0]
             if not result.significant:
                 not_significant.append(results.name)
@@ -372,14 +382,14 @@ def sort_key(item):
         if not self.quiet and not_significant:
             if empty_line:
                 print()
-            self.display_not_signiticant(not_significant)
+            self.display_not_significant(not_significant)
 
-    def compare_suites_list(self):
+    def compare_suites_list(self, all_results):
         not_significant = []
         empty_line = False
         last_index = (len(self.all_results) - 1)
 
-        for index, results in enumerate(self.all_results):
+        for index, results in enumerate(all_results):
             significant = any(result.significant for result in results)
             lines = []
             for result in results:
@@ -406,7 +416,7 @@ def compare_suites_list(self):
         if not self.quiet and not_significant:
             if empty_line:
                 print()
-            self.display_not_signiticant(not_significant)
+            self.display_not_significant(not_significant)
 
     def list_ignored(self):
         for suite, hidden in self.benchmarks.group_by_name_ignored():
@@ -416,9 +426,7 @@ def list_ignored(self):
             print("Ignored benchmarks (%s) of %s: %s"
                   % (len(hidden), suite.filename, ', '.join(sorted(hidden_names))))
 
-    def compare_geometric_mean(self):
-        all_results = self.all_results
-
+    def compare_geometric_mean(self, all_results):
         # use a list since two filenames can be identical,
         # even if results are different
         all_norm_means = []
@@ -443,16 +451,29 @@ def compare_geometric_mean(self):
             geo_mean = format_geometric_mean(all_norm_means[0][1])
             print(f'Geometric mean: {geo_mean}')
 
-    def compare(self):
+    def compare_suites(self, results):
         if self.table:
-            self.compare_suites_table()
+            self.compare_suites_table(results)
         else:
             if self.group_by_speed:
-                self.compare_suites_by_speed()
+                self.compare_suites_by_speed(results)
             else:
-                self.compare_suites_list()
+                self.compare_suites_list(results)
 
-            self.compare_geometric_mean()
+            self.compare_geometric_mean(results)
+
+    def compare(self):
+        if len(self.tags):
+            for tag in self.tags:
+                display_title(f"Benchmarks with tag '{tag}':")
+                all_results = [
+                    results for results in self.all_results
+                    if tag is None or tag in get_tags_for_result(results[0])
+                ]
+                self.compare_suites(all_results)
+                print()
+            display_title(f"All benchmarks:")
+        self.compare_suites(self.all_results)
 
         if not self.quiet:
             self.list_ignored()

pyperf/_metadata.py

@@ -42,6 +42,12 @@ def is_positive(value):
     return (value >= 0)
 
 
+def is_tags(value):
+    if not isinstance(value, list):
+        return False
+    return all(isinstance(x, str) and x not in ('all', '') for x in value)
+
+
 def parse_load_avg(value):
     if isinstance(value, NUMBER_TYPES):
         return value
@@ -62,6 +68,7 @@ def format_noop(value):
 LOOPS = _MetadataInfo(format_number, (int,), is_strictly_positive, 'integer')
 WARMUPS = _MetadataInfo(format_number, (int,), is_positive, 'integer')
 SECONDS = _MetadataInfo(format_seconds, NUMBER_TYPES, is_positive, 'second')
+TAGS = _MetadataInfo(format_generic, (list,), is_tags, 'tag')
 
 # Registry of metadata keys
 METADATA = {
@@ -84,6 +91,7 @@ def format_noop(value):
     'recalibrate_loops': LOOPS,
     'calibrate_warmups': WARMUPS,
     'recalibrate_warmups': WARMUPS,
+    'tags': TAGS,
 }
 
 DEFAULT_METADATA_INFO = _MetadataInfo(format_generic, METADATA_VALUE_TYPES, None, None)