WIP: EqBench: opt, analysis script

eqbench_scripts/analysis

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# Useful functions for analysing EqBench results and to create them.
+
+import argparse
+import csv
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import mkdtemp
+
+
+# List of different comparision which will be done.
+RUNS = [
+     # output dir, disable_patterns, optimalisation, use_build_kernel_opt
+     # ["default-O0-default-patterns", False, "-O0", False],
+     # ["default-O1-default-patterns", False, "-O1", False],
+     ["default-O2-default-patterns", False, "-O2", False],
+     # ["default-O3-default-patterns", False, "-O3", False],
+     # ["default-O0-disabled-patterns", True, "-O0", False],
+     # ["default-O1-disabled-patterns", True, "-O1", False],
+     # ["default-O2-disabled-patterns", True, "-O2", False],
+     # ["default-O3-disabled-patterns", True, "-O3", False],
+     # ["opt-O0-default-patterns", False, "-O0", True],
+     ["opt-O1-default-patterns", False, "-O1", True],
+     # ["opt-O2-default-patterns", False, "-O2", True],
+     # ["opt-O3-default-patterns", False, "-O3", True],
+     # ["opt-O0-disabled-patterns", True, "-O0", True],
+     # ["opt-O1-disabled-patterns", True, "-O1", True],
+     # ["opt-O2-disabled-patterns", True, "-O2", True],
+     # ["opt-O3-disabled-patterns", True, "-O3", True],
+]
+
+
+# Indexes to RUNS items array
+RUN_FILE = 0
+RUN_DISABLE_PATTERNS = 1
+RUN_OPTIMALISATION = 2
+RUN_USE_LLVM_OPT = 3 # Kernel-build llvm passes
+
+
+def compare_multiple(args):
+    """Creates multiple different comparision of EqBench
+    and prints results of the evaluation"""
+    output_dir = args.output_dir
+    eqbench_repo_path = args.eqbench_repo_path
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir)
+    os.mkdir(output_dir)
+    for run in RUNS:
+        command = create_command(
+            output_dir = os.path.join(output_dir, run[RUN_FILE]),
+            eqbench_repo_path=eqbench_repo_path,
+            disable_patterns=run[RUN_DISABLE_PATTERNS],
+            optimalisation=run[RUN_OPTIMALISATION],
+            use_kernel_opt_llvm=run[RUN_USE_LLVM_OPT]
+        )
+        subprocess.check_call(command, stdout=subprocess.DEVNULL,
+                              stderr=subprocess.DEVNULL)
+    get_total(output_dir)
+
+
+def get_total(output_dir):
+    """Prints summary of evaluation for RUNS saved in output_dir."""
+    print("type;opt level;using custom llvm passes;disabled-all-patterns;" +
+          "eq/eq;eq/neq;neq/neq;neq/eq", flush=True)
+    function_level_pattern = re.compile(
+        r"^\| function-level +\| *(\d+) \| +(\d+) \| +(\d+) \| +(\d+) \|$",
+        re.MULTILINE)
+    program_level_pattern = re.compile(
+        r"^\| program-level +\| *(\d+) \| +(\d+) \| +(\d+) \| +(\d+) \|$",
+        re.MULTILINE)
+    for run in RUNS:
+        result_file = os.path.join(output_dir, run[RUN_FILE], "result.md")
+        with open(result_file, "r") as file:
+            lines = file.read()
+            function_level = function_level_pattern.search(lines).groups()
+            program_level = program_level_pattern.search(lines).groups()
+        function_program_total = [str(int(f)+int(p)) for f, p in
+                                  zip(function_level, program_level)]
+        # Note: for CLEVER benchmark and for REVE/triangle is done
+        #       program-level analysis (it was done for this purpose)
+        print(f"function-level;{run[RUN_OPTIMALISATION]};" +
+              f"{'same as build-kernel uses' if run[RUN_USE_LLVM_OPT] else ''};"
+              f"{run[RUN_DISABLE_PATTERNS]};" +
+              f"{';'.join(function_program_total)}", flush=True)
+
+
+def create_command(output_dir, eqbench_repo_path,
+                   disable_patterns=False, optimalisation="-O0",
+                   use_build_kernel_opt_llvm=False):
+    """Creates command for running EqBench benchmark (with eqbench_scripts/run)
+    :param output_dir: path where to save results
+    :param eqbench_repo_path: path to EqBench sources
+    :param disable_patterns: true to disable all patterns
+    :param optimalisation: level of optimalisation for compilation
+    :param use_build_kernel_opt_llvm: use passes which are used for build-kernel
+    """
+    run_script = Path(Path(__file__).parent, "run")
+    command = [run_script, eqbench_repo_path, "-o",
+               output_dir,
+               "--output-src-file-paths"]
+    if disable_patterns:
+        command.append("--disable-patterns")
+    if optimalisation:
+        command.append(f"--add-clang-options={optimalisation}")
+    if use_build_kernel_opt_llvm:
+        command.append("--build-kernel-opt-llvm")
+    return command
+
+
+def get_incorect(output_dir):
+    """Returns csv optput with all incorectly evaluated programs
+    from all runs saved in output_dir"""
+    TYPE = 0
+    CORRECT = 5
+    incorrect = set()
+    for run in RUNS:
+        results_path = \
+            os.path.join(output_dir, run[RUN_FILE], "eqbench-results.csv")
+        with open(results_path, "r") as file:
+            reader = csv.reader(file, delimiter=";")
+            head = next(reader)
+            for line in reader:
+                if line[TYPE] in ["function-level", "program-level"] \
+                   and line[CORRECT] == "False":
+                    incorrect.add(
+                                #   run[RUN_FILE]+";" +
+                                  ";".join(line))
+    print(";".join(head))
+    for line in incorrect:
+        print(line)
+
+
+def diff_2_multiruns(args):
+    """Show differences between 2 saved outputs of compare-multiple command"""
+    dir1 = args.old_run_dir
+    dir2 = args.new_run_dir
+    tmp_dir = mkdtemp()
+    dir1_tmp = Path(tmp_dir, Path(dir1).name)
+    dir1_tmp.mkdir()
+    dir2_tmp = Path(tmp_dir, Path(dir2).name)
+    dir2_tmp.mkdir()
+    for run in RUNS:
+        file1 = str(Path(dir1, run[RUN_FILE], "eqbench-results.csv"))
+        file2 = str(Path(dir2, run[RUN_FILE], "eqbench-results.csv"))
+        output1 = str(Path(dir1_tmp, run[RUN_FILE]).with_suffix(".csv"))
+        output2 = str(Path(dir2_tmp, run[RUN_FILE]).with_suffix(".csv"))
+        # fixme use popen
+        p1 = subprocess.Popen(("cut",f"{file1}", "-d", ";", "-f", "1-6"), stdout=subprocess.PIPE)
+        subprocess.check_call(["sort", "-o", f"{output1}"], stdin=p1.stdout)
+        p1.wait()
+        p2 = subprocess.Popen(("cut",f"{file2}", "-d", ";", "-f", "1-6"), stdout=subprocess.PIPE)
+        subprocess.check_call(["sort", "-o", f"{output2}"], stdin=p2.stdout)
+        p2.wait()
+    os.system(f"meld {str(dir1_tmp)} {str(dir2_tmp)}")
+    shutil.rmtree(tmp_dir)
+
+def diff_2_runs(args):
+    """Show differense between 2 runs of `run` program."""
+    dir1 = args.old_run_dir
+    dir2 = args.new_run_dir
+    tmp_dir = mkdtemp()
+    file1 = str(Path(dir1, "eqbench-results.csv"))
+    file2 = str(Path(dir2, "eqbench-results.csv"))
+    output1 = str(Path(tmp_dir, Path(dir1).name))
+    output2 = str(Path(tmp_dir, Path(dir2).name))
+    p1 = subprocess.Popen(("cut",f"{file1}", "-d", ";", "-f", "1-6"), stdout=subprocess.PIPE)
+    subprocess.check_call(["sort", "-o", f"{output1}"], stdin=p1.stdout)
+    p1.wait()
+    p2 = subprocess.Popen(("cut",f"{file2}", "-d", ";", "-f", "1-6"), stdout=subprocess.PIPE)
+    subprocess.check_call(["sort", "-o", f"{output2}"], stdin=p2.stdout)
+    p2.wait()
+    os.system(f"meld {str(output1)} {str(output2)}")
+    shutil.rmtree(tmp_dir)
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="EqBench analyzator")
+    subcommands = parser.add_subparsers()
+
+    cmp_mul = subcommands.add_parser("compare-multiple",
+                                     help="Compares EqBench " +
+                                     "programs with different options.")
+    cmp_mul.add_argument("output_dir", help="directory to save outputs")
+    cmp_mul.add_argument("eqbench_repo_path", help="path to EqBench sources")
+    cmp_mul.set_defaults(func=compare_multiple)
+
+    diff2m = subcommands.add_parser("diff-2-multiruns", help="show diffs " +
+                                   "of two outputs of `compare multiple` command")
+    diff2m.add_argument("old_run_dir")
+    diff2m.add_argument("new_run_dir")
+    diff2m.set_defaults(func=diff_2_multiruns)
+
+    diff2r = subcommands.add_parser("diff-2-runs", help="Show diff " +
+                                   "of 2 outputs of `run` program.")
+    diff2r.add_argument("old_run_dir")
+    diff2r.add_argument("new_run_dir")
+    diff2r.set_defaults(func=diff_2_runs)
+
+
+    total = subcommands.add_parser("get-total", help="Get summary " +
+                                   "of `compare-multiple` command.")
+    total.add_argument("multiple_runs_dir")
+    total.set_defaults(func=lambda args: get_total(args.multiple_runs_dir))
+
+    args = parser.parse_args()
+    args.func(args)

eqbench_scripts/opt

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+FOLDER="eqbench-results-23.10.23-f0bb5e2a4909f5078d/opt-O1-default-patterns"
+EXPECTED="Eq"
+PASSES="instcombine"
+# PASSES="ipsccp,reassociate,sroa,instcombine,sccp"
+# PASSES="verify,annotation2metadata,forceattrs,inferattrs,function<eager-inv>(lower-expect,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;no-switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;no-sink-common-insts>,sroa,early-cse<>,coro-early),openmp-opt,ipsccp,called-value-propagation,globalopt,function(mem2reg),deadargelim,function<eager-inv>(instcombine,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;no-sink-common-insts>),require<globals-aa>,function(invalidate<aa>),require<profile-summary>,cgscc(devirt<4>(inline<only-mandatory>,inline,function-attrs,openmp-opt-cgscc,function<eager-inv>(sroa,early-cse<memssa>,speculative-execution,jump-threading,correlated-propagation,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;no-sink-common-insts>,instcombine,libcalls-shrinkwrap,tailcallelim,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;no-sink-common-insts>,reassociate,require<opt-remark-emit>,loop-mssa(loop-instsimplify,loop-simplifycfg,licm,loop-rotate,licm,simple-loop-unswitch<no-nontrivial;trivial>),simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;no-sink-common-insts>,instcombine,loop(loop-idiom,indvars,loop-deletion,loop-unroll-full),sroa,mldst-motion<no-split-footer-bb>,gvn<>,sccp,bdce,instcombine,jump-threading,correlated-propagation,adce,memcpyopt,dse,loop-mssa(licm),coro-elide,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;hoist-common-insts;sink-common-insts>,instcombine),coro-split)),globalopt,globaldce,elim-avail-extern,rpo-function-attrs,require<globals-aa>,function<eager-inv>(float2int,lower-constant-intrinsics,loop(loop-rotate,loop-deletion),loop-distribute,inject-tli-mappings,loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>,loop-load-elim,instcombine,simplifycfg<bonus-inst-threshold=1;forward-switch-cond;switch-range-to-icmp;switch-to-lookup;no-keep-loops;hoist-common-insts;sink-common-insts>,slp-vectorizer,vector-combine,instcombine,loop-unroll<O2>,transform-warning,instcombine,require<opt-remark-emit>,loop-mssa(licm),alignment-from-assumptions,loop-sink,instsimplify,div-rem-pairs,simplifycfg<bonus-inst-threshold=1;no-forward-switch-cond;switch-range-to-icmp;no-switch-to-lookup;keep-loops;no-hoist-common-insts;no-sink-common-insts>,coro-cleanup),cg-profile,globaldce,constmerge,rel-lookup-table-converter,function(annotation-remarks),verify"
+tmp=$(mktemp -d /tmp/snapshots.XXX)
+
+
+cp -r ${FOLDER}/benchmarks-build/${1}/${2}/${EXPECTED}/old $tmp
+cp -r ${FOLDER}/benchmarks-build/${1}/${2}/${EXPECTED}/new $tmp
+
+rm ${tmp}/old/old*-simpl.ll
+rm ${tmp}/new/new*-simpl.ll
+
+oldll=`realpath ${tmp}/old/oldV.ll`
+newll=`realpath ${tmp}/new/newV.ll`
+opt -passes "${PASSES}" ${oldll} -S -o $oldll
+opt -passes "${PASSES}" ${newll} -S -o $newll
+result/bin/diffkemp -ddd compare $tmp/old $tmp/new --output-llvm-ir --stdout &
+meld ${tmp}/old/old*.c ${tmp}/new/new*.c &
+
+# ll vm file
+# cat ${FOLDER}/benchmarks-build/${1}/${2}/${EXPECTED}/old/old*.ll > $old
+# cat ${FOLDER}/benchmarks-build/${1}/${2}/${EXPECTED}/new/new*.ll > $new
+
+# simpll file
+# cat ${FOLDER}/benchmarks-build/${1}/${2}/Eq/old/old*-simpl.ll > $old
+# cat ${FOLDER}/benchmarks-build/${1}/${2}/Eq/new/new*-simpl.ll > $new
+
+# simpll file but cleaner
+# ( egrep -v "^.*call void @llvm\.dbg\..*$" ${tmp}/old/oldV-simpl.ll | egrep -v "^.*@log.*$" > ${tmp}/old.simpll ) ; \
+# ( egrep -v "^.*call void @llvm\.dbg\..*$" ${tmp}/new/newV-simpl.ll | egrep -v "^.*@log.*$" > ${tmp}/new.simpll ) ; \
+meld ${tmp}/old/oldV-simpl.ll ${tmp}/new/newV-simpl.ll
+# meld ${oldll} ${newll}
+