From 46ad8ac998a41d4beaff2d343a61ac7de46ec467 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 22 Dec 2020 18:27:41 +0300 Subject: [PATCH 1/7] Compare costs in debug format as well. --- .../emu-cost/compare_costs.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/runtime/runtime-params-estimator/emu-cost/compare_costs.py b/runtime/runtime-params-estimator/emu-cost/compare_costs.py index a7e667efbb4..a8e5905cbe0 100755 --- a/runtime/runtime-params-estimator/emu-cost/compare_costs.py +++ b/runtime/runtime-params-estimator/emu-cost/compare_costs.py @@ -25,11 +25,30 @@ def flatten_dict(d, result, prefix=''): return result +def parse_debug_print(path): + result = {} + with open(path) as f: + pattern1 = re.compile("\\s*\"*([\\w]+)\"*: ([\\d]+).*") + pattern2 = re.compile("\\s*\"*([\\w]+)\"*:.*") + prefix = "" + for line in f: + m = pattern1.search(line) + if m != None: + result[prefix + m.group(1)] = m.group(2) + else: + m = pattern2.search(line) + if m != None: + prefix = m.group(1) + ": " + return result def read_costs(path): result = OrderedDict() - with open(path) as f: - genesis_or_runtime_config = json.load(f, object_pairs_hook=OrderedDict) + try: + with open(path) as f: + genesis_or_runtime_config = json.load(f, object_pairs_hook=OrderedDict) + except (json.decoder.JSONDecodeError): + # try to load Rust debug pretty print. + genesis_or_runtime_config = parse_debug_print(path) if 'runtime_config' in genesis_or_runtime_config: runtime_config = genesis_or_runtime_config['runtime_config'] else: From 59ce53239013c3eadd12d776fa24b0efdbc02427 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 22 Dec 2020 18:28:12 +0300 Subject: [PATCH 2/7] Take IO costs into account. --- .../runtime-params-estimator/src/testbed_runners.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/runtime/runtime-params-estimator/src/testbed_runners.rs b/runtime/runtime-params-estimator/src/testbed_runners.rs index b723a9989cf..0d2a9728a97 100644 --- a/runtime/runtime-params-estimator/src/testbed_runners.rs +++ b/runtime/runtime-params-estimator/src/testbed_runners.rs @@ -109,9 +109,14 @@ pub fn measure_actions( measure_transactions(metric, measurements, config, testbed, &mut f, false) } +// We use several "magical" file descriptors to interact with the plugin in QEMU +// intercepting read syscall. Plugin counts instructions executed and amount of data transferred +// by IO operations. We "normalize" all those costs into instruction count. const CATCH_BASE: u32 = 0xcafebabe; const HYPERCALL_START_COUNTING: u32 = 0; const HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED: u32 = 1; +const HYPERCALL_GET_BYTES_READ: u32 = 2; +const HYPERCALL_GET_BYTES_WRITTEN: u32 = 3; fn hypercall(index: u32) -> u64 { let mut result: u64 = 0; @@ -132,7 +137,11 @@ fn start_count_instructions() -> Consumed { } fn end_count_instructions() -> u64 { - hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED) + let result_insn = hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED); + let result_read = hypercall(HYPERCALL_GET_BYTES_READ); + let result_written = hypercall(HYPERCALL_GET_BYTES_WRITTEN); + // See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values. + result_insn + result_read * 27 + result_written * 47 } fn start_count_time() -> Consumed { From 75fbe0b7819fa6741940426e38c72ed4e01c81d5 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 22 Dec 2020 18:37:20 +0300 Subject: [PATCH 3/7] Allow simpler selection of costs model. --- .../src/testbed_runners.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/runtime/runtime-params-estimator/src/testbed_runners.rs b/runtime/runtime-params-estimator/src/testbed_runners.rs index 0d2a9728a97..d332ea647ad 100644 --- a/runtime/runtime-params-estimator/src/testbed_runners.rs +++ b/runtime/runtime-params-estimator/src/testbed_runners.rs @@ -137,11 +137,16 @@ fn start_count_instructions() -> Consumed { } fn end_count_instructions() -> u64 { - let result_insn = hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED); - let result_read = hypercall(HYPERCALL_GET_BYTES_READ); - let result_written = hypercall(HYPERCALL_GET_BYTES_WRITTEN); - // See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values. - result_insn + result_read * 27 + result_written * 47 + const use_io_costs: bool = true; + if use_io_costs { + let result_insn = hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED); + let result_read = hypercall(HYPERCALL_GET_BYTES_READ); + let result_written = hypercall(HYPERCALL_GET_BYTES_WRITTEN); + // See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values. + result_insn + result_read * 27 + result_written * 47 + } else { + hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED) + } } fn start_count_time() -> Consumed { From f1b4c5ef78fdde1dc3182ed1b9a1f07bf91350b3 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 22 Dec 2020 21:34:19 +0300 Subject: [PATCH 4/7] Fix warnings --- runtime/runtime-params-estimator/src/testbed_runners.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/runtime-params-estimator/src/testbed_runners.rs b/runtime/runtime-params-estimator/src/testbed_runners.rs index d332ea647ad..142cdd9ef0f 100644 --- a/runtime/runtime-params-estimator/src/testbed_runners.rs +++ b/runtime/runtime-params-estimator/src/testbed_runners.rs @@ -137,8 +137,8 @@ fn start_count_instructions() -> Consumed { } fn end_count_instructions() -> u64 { - const use_io_costs: bool = true; - if use_io_costs { + const USE_IO_COSTS: bool = true; + if USE_IO_COSTS { let result_insn = hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED); let result_read = hypercall(HYPERCALL_GET_BYTES_READ); let result_written = hypercall(HYPERCALL_GET_BYTES_WRITTEN); From 36374f8f3e227be43742d2392482da30795f9c6a Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Wed, 23 Dec 2020 18:59:38 +0300 Subject: [PATCH 5/7] Implement diff option to compare script. --- .../emu-cost/compare_costs.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/runtime/runtime-params-estimator/emu-cost/compare_costs.py b/runtime/runtime-params-estimator/emu-cost/compare_costs.py index a8e5905cbe0..8b1c0c1a87a 100755 --- a/runtime/runtime-params-estimator/emu-cost/compare_costs.py +++ b/runtime/runtime-params-estimator/emu-cost/compare_costs.py @@ -61,16 +61,22 @@ def rate(c2, c1): return "n/a" return '{:.2f}'.format(float(c2) / float(c1)) +EPSILON=0.2 +def significant(c1, c2): + if c1 == 0 or c2 == 0: + return c1 != c2 + return abs((c1 / c2) - 1.0) > EPSILON or abs((c2 / c1) - 1.0) > EPSILON -def process_props(file1, file2, safety1, safety2): +def process_props(file1, file2, safety1, safety2, diff): costs1 = read_costs(file1) costs2 = read_costs(file2) for key in costs1: c1 = int(costs1[key]) * safety1 c2 = int(costs2.get(key, "0")) * safety2 - print("{}: first={} second={} second/first={}".format( - key, c1, c2, rate(c2, c1))) + if not diff or significant(c1, c2): + print("{}: first={} second={} second/first={}".format( + key, c1, c2, rate(c2, c1))) def process_json(file1, file2): @@ -92,7 +98,9 @@ def process_json(file1, file2): parser.add_argument('--safety_second', default=1, help='Safety multiplier applied to second') + parser.add_argument('--diff', dest='diff', action='store_true') + parser.set_defaults(diff=False) args = parser.parse_args() process_props(args.files[0], args.files[1], int(args.safety_first), - int(args.safety_second)) + int(args.safety_second), args.diff) From 64e474023058ab9280c638361eb94132084a9381 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Fri, 15 Jan 2021 22:27:45 +0300 Subject: [PATCH 6/7] Review feedback --- runtime/runtime-params-estimator/src/testbed_runners.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/runtime/runtime-params-estimator/src/testbed_runners.rs b/runtime/runtime-params-estimator/src/testbed_runners.rs index 142cdd9ef0f..8b180376a8d 100644 --- a/runtime/runtime-params-estimator/src/testbed_runners.rs +++ b/runtime/runtime-params-estimator/src/testbed_runners.rs @@ -118,6 +118,10 @@ const HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED: u32 = 1; const HYPERCALL_GET_BYTES_READ: u32 = 2; const HYPERCALL_GET_BYTES_WRITTEN: u32 = 3; +// See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values. +const READ_BYTE_COST: u32 = 27; +const WRITE_BYTE_COST: u32 = 47; + fn hypercall(index: u32) -> u64 { let mut result: u64 = 0; unsafe { @@ -142,8 +146,8 @@ fn end_count_instructions() -> u64 { let result_insn = hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED); let result_read = hypercall(HYPERCALL_GET_BYTES_READ); let result_written = hypercall(HYPERCALL_GET_BYTES_WRITTEN); - // See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values. - result_insn + result_read * 27 + result_written * 47 + + result_insn + result_read * READ_BYTE_COST + result_written * WRITE_BYTE_COST } else { hypercall(HYPERCALL_STOP_AND_GET_INSTRUCTIONS_EXECUTED) } From a1ccba054f7781152c71fa44365b122488aec7a6 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Fri, 15 Jan 2021 22:42:02 +0300 Subject: [PATCH 7/7] Typo. --- runtime/runtime-params-estimator/src/testbed_runners.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/runtime-params-estimator/src/testbed_runners.rs b/runtime/runtime-params-estimator/src/testbed_runners.rs index 8b180376a8d..e4749ace45b 100644 --- a/runtime/runtime-params-estimator/src/testbed_runners.rs +++ b/runtime/runtime-params-estimator/src/testbed_runners.rs @@ -119,8 +119,8 @@ const HYPERCALL_GET_BYTES_READ: u32 = 2; const HYPERCALL_GET_BYTES_WRITTEN: u32 = 3; // See runtime/runtime-params-estimator/emu-cost/README.md for the motivation of constant values. -const READ_BYTE_COST: u32 = 27; -const WRITE_BYTE_COST: u32 = 47; +const READ_BYTE_COST: u64 = 27; +const WRITE_BYTE_COST: u64 = 47; fn hypercall(index: u32) -> u64 { let mut result: u64 = 0;