Take IO costs into account

runtime/runtime-params-estimator/emu-cost/compare_costs.py

@@ -25,11 +25,30 @@ def flatten_dict(d, result, prefix=''):
 
     return result
 
+def parse_debug_print(path):
+    result = {}
+    with open(path) as f:
+        pattern1 = re.compile("\\s*\"*([\\w]+)\"*: ([\\d]+).*")
+        pattern2 = re.compile("\\s*\"*([\\w]+)\"*:.*")
+        prefix = ""
+        for line in f:
+            m = pattern1.search(line)
+            if m != None:
+                result[prefix + m.group(1)] = m.group(2)
+            else:
+                m = pattern2.search(line)
+                if m != None:
+                    prefix = m.group(1) + ": "
+    return result
 
 def read_costs(path):
     result = OrderedDict()
-    with open(path) as f:
-        genesis_or_runtime_config = json.load(f, object_pairs_hook=OrderedDict)
+    try:
+        with open(path) as f:
+            genesis_or_runtime_config = json.load(f, object_pairs_hook=OrderedDict)
+    except (json.decoder.JSONDecodeError):
+        # try to load Rust debug pretty print.
+        genesis_or_runtime_config = parse_debug_print(path)
     if 'runtime_config' in genesis_or_runtime_config:
         runtime_config = genesis_or_runtime_config['runtime_config']
     else:
@@ -42,16 +61,22 @@ def rate(c2, c1):
         return "n/a"
     return '{:.2f}'.format(float(c2) / float(c1))
 
+EPSILON=0.2
+def significant(c1, c2):
+    if c1 == 0 or c2 == 0:
+        return c1 != c2
+    return abs((c1 / c2) - 1.0) > EPSILON or abs((c2 / c1) - 1.0) > EPSILON
 
-def process_props(file1, file2, safety1, safety2):
+def process_props(file1, file2, safety1, safety2, diff):
     costs1 = read_costs(file1)
     costs2 = read_costs(file2)
 
     for key in costs1:
         c1 = int(costs1[key]) * safety1
         c2 = int(costs2.get(key, "0")) * safety2
-        print("{}: first={} second={} second/first={}".format(
-            key, c1, c2, rate(c2, c1)))
+        if not diff or significant(c1, c2):
+            print("{}: first={} second={} second/first={}".format(
+                key, c1, c2, rate(c2, c1)))
 
 
 def process_json(file1, file2):
@@ -73,7 +98,9 @@ def process_json(file1, file2):
     parser.add_argument('--safety_second',
                         default=1,
                         help='Safety multiplier applied to second')
+    parser.add_argument('--diff', dest='diff', action='store_true')
+    parser.set_defaults(diff=False)
     args = parser.parse_args()
 
     process_props(args.files[0], args.files[1], int(args.safety_first),
-                  int(args.safety_second))
+                  int(args.safety_second), args.diff)

runtime/runtime-params-estimator/src/testbed_runners.rs

@@ -109,9 +109,14 @@ pub fn measure_actions(
     measure_transactions(metric, measurements, config, testbed, &mut f, false)
 }
 
+// We use several "magical" file descriptors to interact with the plugin in QEMU
+// intercepting read syscall. Plugin counts instructions executed and amount of data transferred
+// by IO operations. We "normalize" all those costs into instruction count.
 const CATCH_BASE: u32 = 0xcafebabe;
 const HYPERCALL_START_COUNTING: u32 = 0;
 const HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED: u32 = 1;
+const HYPERCALL_GET_BYTES_READ: u32 = 2;
+const HYPERCALL_GET_BYTES_WRITTEN: u32 = 3;
 
 fn hypercall(index: u32) -> u64 {
     let mut result: u64 = 0;
@@ -132,7 +137,16 @@ fn start_count_instructions() -> Consumed {
 }
 
 fn end_count_instructions() -> u64 {
-    hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED)
+    const USE_IO_COSTS: bool = true;
+    if USE_IO_COSTS {
+        let result_insn = hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED);
+        let result_read = hypercall(HYPERCALL_GET_BYTES_READ);
+        let result_written = hypercall(HYPERCALL_GET_BYTES_WRITTEN);
+        // See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values.
+        result_insn + result_read * 27 + result_written * 47
+    } else {
+        hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED)
+    }
 }
 
 fn start_count_time() -> Consumed {