diff --git a/src/consistency-testing/gobekli/bin/gobekli-report b/src/consistency-testing/gobekli/bin/gobekli-report index 688f3b863ae77..0368c11f50c87 100755 --- a/src/consistency-testing/gobekli/bin/gobekli-report +++ b/src/consistency-testing/gobekli/bin/gobekli-report @@ -9,8 +9,10 @@ from pathlib import Path from gobekli.logging import m from gobekli.chaos.analysis import (make_overview_chart, make_latency_chart, - make_pdf_latency_chart, make_availability_chart, - analyze_inject_recover_availability, LatencyType) + make_pdf_latency_chart, + make_availability_chart, + analyze_inject_recover_availability, + LatencyType) from os import path import json @@ -242,22 +244,49 @@ EXPERIMENT = """ """ + def build_charts(config, root, results, warmup_s, zoom_us): for result in results: path = os.path.join(root, result["path"]) - make_overview_chart(result["title"], path, result["availability_log"], result["latency_log"], warmup_s, LatencyType.OVERALL) - make_overview_chart(result["title"], path, result["availability_log"], result["latency_log"], warmup_s, LatencyType.PRODUCER) - make_availability_chart(result["title"], None, path, result["availability_log"], result["latency_log"], warmup_s) + make_overview_chart(result["title"], path, result["availability_log"], + result["latency_log"], warmup_s, + LatencyType.OVERALL) + make_overview_chart(result["title"], path, result["availability_log"], + result["latency_log"], warmup_s, + LatencyType.PRODUCER) + make_availability_chart(result["title"], None, path, + result["availability_log"], + result["latency_log"], warmup_s) for endpoint in config["endpoints"]: - make_availability_chart(result["title"], endpoint["idx"], path, result["availability_log"], result["latency_log"], warmup_s) - make_pdf_latency_chart(result["title"], None, path, result["availability_log"], result["latency_log"], warmup_s, zoom_us, LatencyType.OVERALL) - make_pdf_latency_chart(result["title"], None, path, result["availability_log"], result["latency_log"], warmup_s, zoom_us, LatencyType.PRODUCER) + make_availability_chart(result["title"], endpoint["idx"], path, + result["availability_log"], + result["latency_log"], warmup_s) + make_pdf_latency_chart(result["title"], None, path, + result["availability_log"], + result["latency_log"], warmup_s, zoom_us, + LatencyType.OVERALL) + make_pdf_latency_chart(result["title"], None, path, + result["availability_log"], + result["latency_log"], warmup_s, zoom_us, + LatencyType.PRODUCER) for endpoint in config["endpoints"]: - make_pdf_latency_chart(result["title"], endpoint["idx"], path, result["availability_log"], result["latency_log"], warmup_s, zoom_us, LatencyType.OVERALL) - make_pdf_latency_chart(result["title"], endpoint["idx"], path, result["availability_log"], result["latency_log"], warmup_s, zoom_us, LatencyType.PRODUCER) + make_pdf_latency_chart(result["title"], endpoint["idx"], path, + result["availability_log"], + result["latency_log"], warmup_s, zoom_us, + LatencyType.OVERALL) + make_pdf_latency_chart(result["title"], endpoint["idx"], path, + result["availability_log"], + result["latency_log"], warmup_s, zoom_us, + LatencyType.PRODUCER) for endpoint in config["endpoints"]: - make_latency_chart(result["title"], endpoint["idx"], path, result["availability_log"], result["latency_log"], warmup_s, LatencyType.OVERALL) - make_latency_chart(result["title"], endpoint["idx"], path, result["availability_log"], result["latency_log"], warmup_s, LatencyType.PRODUCER) + make_latency_chart(result["title"], endpoint["idx"], path, + result["availability_log"], + result["latency_log"], warmup_s, + LatencyType.OVERALL) + make_latency_chart(result["title"], endpoint["idx"], path, + result["availability_log"], + result["latency_log"], warmup_s, + LatencyType.PRODUCER) came_from = os.getcwd() os.chdir(path) gnuplot("pdf.latency.overall.all.gp") @@ -274,7 +303,7 @@ def build_charts(config, root, results, warmup_s, zoom_us): gnuplot(f"pdf.latency.producer.{idx}.gp") rm(f"pdf.latency.producer.{idx}.gp") rm(f"pdf.latency.producer.{idx}.log") - + gnuplot("availability.all.gp") rm("availability.all.gp") rm("availability.all.log") @@ -304,6 +333,7 @@ def build_charts(config, root, results, warmup_s, zoom_us): os.chdir(came_from) + def archive_logs(root, results): logs = [] for result in results: @@ -315,9 +345,10 @@ def archive_logs(root, results): for f in logs: os.remove(f) + def archive_failed_cmd_log(root, results): for result in results: - status = result["status"] # passed | failed + status = result["status"] # passed | failed if status == "passed": for f in os.listdir(path.join(root, result["path"])): if f.startswith(result["cmd_log"]): @@ -325,7 +356,8 @@ def archive_failed_cmd_log(root, results): else: cmd_logs = [] for f in os.listdir(path.join(root, result["path"])): - if f.startswith(result["cmd_log"]) and not f.endswith(".tar.bz2"): + if f.startswith( + result["cmd_log"]) and not f.endswith(".tar.bz2"): cmd_logs.append(f) if len(cmd_logs) > 0: tar_args = ["cjf", result["cmd_log"] + ".tar.bz2"] + cmd_logs @@ -338,7 +370,8 @@ def archive_failed_cmd_log(root, results): class ChartSet: - def __init__(self, id, title, latency_overall, latency_producer, pdf_latency_overall, pdf_latency_producer, availability): + def __init__(self, id, title, latency_overall, latency_producer, + pdf_latency_overall, pdf_latency_producer, availability): self.title = title self.id = id self.latency_overall = latency_overall @@ -347,36 +380,41 @@ class ChartSet: self.pdf_latency_producer = pdf_latency_producer self.availability = availability + def build_experiment_index(context, config, root, result, warmup, zoom_us): index_path = os.path.join(root, result["path"], "index.html") - + charts = [] if len(config["endpoints"]) > 1: charts.append( - ChartSet("overview", "Combined", "overview.overall.png", "overview.producer.png", "pdf.latency.overall.all.png", "pdf.latency.producer.all.png", "availability.all.png") - ) + ChartSet("overview", "Combined", "overview.overall.png", + "overview.producer.png", "pdf.latency.overall.all.png", + "pdf.latency.producer.all.png", "availability.all.png")) for endpoint in config["endpoints"]: idx = endpoint["idx"] charts.append( - ChartSet(endpoint["id"], endpoint["id"], f"latency.overall.{idx}.png", f"latency.producer.{idx}.png", f"pdf.latency.overall.{idx}.png", f"pdf.latency.producer.{idx}.png", f"availability.{idx}.png") - ) - + ChartSet(endpoint["id"], endpoint["id"], + f"latency.overall.{idx}.png", + f"latency.producer.{idx}.png", + f"pdf.latency.overall.{idx}.png", + f"pdf.latency.producer.{idx}.png", + f"availability.{idx}.png")) - with open(index_path, 'w') as html: - html.write(jinja2.Template(EXPERIMENT).render( - system = context["system"], - workload = context["workload"], - scenario = context["scenario"], - charts = charts, - fault = result["fault"], - id = result["id"], - min_lat = result["stat"]["min_lat"], - max_lat = result["stat"]["max_lat"], - max_unavailability = result["stat"]["max_unavailability"] - )) + html.write( + jinja2.Template(EXPERIMENT).render( + system=context["system"], + workload=context["workload"], + scenario=context["scenario"], + charts=charts, + fault=result["fault"], + id=result["id"], + min_lat=result["stat"]["min_lat"], + max_lat=result["stat"]["max_lat"], + max_unavailability=result["stat"]["max_unavailability"])) + def load_results(context, results_log, warmup_s): root = Path(results_log).parent @@ -384,42 +422,51 @@ def load_results(context, results_log, warmup_s): for line in result_file: result = json.loads(line) result["stat"] = analyze_inject_recover_availability( - path.join(root, result["path"]), - result["availability_log"], - result["latency_log"], - warmup_s - ) + path.join(root, result["path"]), result["availability_log"], + result["latency_log"], warmup_s) yield result + def load_context(root): with open(path.join(root, "context.json")) as context_info: return json.load(context_info) + def build_alerts(root, results): with open(path.join(root, "alerts.log"), "w") as alerts: for result in results: if result["status"] == "failed": - alerts.write(str(m(type="consistency", message=result["error"], id=result["id"])) + "\n") + alerts.write( + str( + m(type="consistency", + message=result["error"], + id=result["id"])) + "\n") + def load_config(root): with open(path.join(root, "settings.json")) as config_json: return json.load(config_json) + def build_index(context, title, root, results): ava_stat = defaultdict(lambda: []) - fault_stat = defaultdict(lambda: { "passed": 0, "failed": [] }) - + fault_stat = defaultdict(lambda: {"passed": 0, "failed": []}) + for result in results: fault = result["fault"] - status = result["status"] # passed | failed + status = result["status"] # passed | failed if status == "passed": fault_stat[fault]["passed"] += 1 else: fault_stat[fault]["failed"].append({ - "error": result["error"], - "status": "failed", - "logs": path.join(result["path"], result["cmd_log"] + ".tar.bz2"), - "is_err": True + "error": + result["error"], + "status": + "failed", + "logs": + path.join(result["path"], result["cmd_log"] + ".tar.bz2"), + "is_err": + True }) for result in results: @@ -427,13 +474,15 @@ def build_index(context, title, root, results): availability["id"] = result["id"] availability["link"] = os.path.join(result["path"], "index.html") ava_stat[result["fault"]].append(availability) - + atests = [] for fault in ava_stat.keys(): stat = ava_stat[fault] atests.append({ - "fault": fault, "span": len(stat), - "first": stat[0], "rest": stat[1:] + "fault": fault, + "span": len(stat), + "first": stat[0], + "rest": stat[1:] }) ctests = [] @@ -451,34 +500,41 @@ def build_index(context, title, root, results): first = fault_stat[fault]["failed"][0] rest = fault_stat[fault]["failed"][1:] - ctests.append({"fault": fault, "span": 1 + len(rest), "first": first, "rest": rest}) + ctests.append({ + "fault": fault, + "span": 1 + len(rest), + "first": first, + "rest": rest + }) with open(path.join(root, "index.html"), 'w') as html: - html.write(jinja2.Template(INDEX).render( - title = title, - system = context["system"], - workload = context["workload"], - scenario = context["scenario"], - ctests = ctests, - atests = atests - )) + html.write( + jinja2.Template(INDEX).render(title=title, + system=context["system"], + workload=context["workload"], + scenario=context["scenario"], + ctests=ctests, + atests=atests)) + def build_report(results_log, warmup_s, zoom_us): root = Path(results_log).parent context = load_context(root) config = load_config(root) - results = list(load_results(context, results_log, warmup_s)) + results = list(load_results(context, results_log, warmup_s)) build_charts(config, root, results, warmup_s, zoom_us) archive_failed_cmd_log(root, results) - + for result in results: - build_experiment_index(context, config, root, result, warmup_s, zoom_us) - + build_experiment_index(context, config, root, result, warmup_s, + zoom_us) + build_index(context, results_log, root, results) build_alerts(root, results) archive_logs(root, results) + parser = argparse.ArgumentParser(description='build gobekli report') parser.add_argument('--result', action='append', required=True) parser.add_argument('--warmup_s', type=int, default=5, required=False) @@ -486,4 +542,4 @@ parser.add_argument('--zoom_us', type=int, default=100000, required=False) args = parser.parse_args() for result in args.result: - build_report(result, args.warmup_s, args.zoom_us) \ No newline at end of file + build_report(result, args.warmup_s, args.zoom_us) diff --git a/src/consistency-testing/gobekli/bin/gobekli-run b/src/consistency-testing/gobekli/bin/gobekli-run index 75f374ce6c5ce..1b9047f29138a 100644 --- a/src/consistency-testing/gobekli/bin/gobekli-run +++ b/src/consistency-testing/gobekli/bin/gobekli-run @@ -8,48 +8,82 @@ from gobekli.kvapi import KVNode from gobekli.workloads import (symmetrical_mrsw, symmetrical_comrmw) from gobekli.logging import init_logs -async def mrsw_aio(kvs, writers, readers, duration, ss_metrics, cmd_log, latency_log, stat_log): - if ss_metrics==None: + +async def mrsw_aio(kvs, writers, readers, duration, ss_metrics, cmd_log, + latency_log, stat_log): + if ss_metrics == None: ss_metrics = [] init_logs(cmd_log, latency_log, stat_log, ss_metrics) - + nodes = [] for i in range(0, len(kvs)): nodes.append(KVNode(i, kvs[i], kvs[i])) - - await symmetrical_mrsw.start_mrsw_workload_aio(nodes, writers, readers, duration, ss_metrics) - + + await symmetrical_mrsw.start_mrsw_workload_aio(nodes, writers, readers, + duration, ss_metrics) + for kv in nodes: await kv.close_aio() -async def comrmw_aio(kvs, writers, readers, duration, ss_metrics, cmd_log, latency_log, stat_log): - if ss_metrics==None: + +async def comrmw_aio(kvs, writers, readers, duration, ss_metrics, cmd_log, + latency_log, stat_log): + if ss_metrics == None: ss_metrics = [] init_logs(cmd_log, latency_log, stat_log, ss_metrics) - + nodes = [] for i in range(0, len(kvs)): nodes.append(KVNode(i, kvs[i], kvs[i])) - - await symmetrical_comrmw.start_comrsw_workload_aio(nodes, writers, readers, duration, ss_metrics) - + + await symmetrical_comrmw.start_comrsw_workload_aio(nodes, writers, readers, + duration, ss_metrics) + for kv in nodes: await kv.close_aio() + parser = argparse.ArgumentParser(description='Gobekli') parser.add_argument('workload', choices=['mrsw', 'comrsw']) parser.add_argument('--kv', action='append', required=True) -parser.add_argument('--cmd-log', default="cmd.log", required=False, help="command log") -parser.add_argument('--latency-log', default="latency.log", required=False, help="latency log") -parser.add_argument('--stat-log', default="stat.log", required=False, help="stat log") -parser.add_argument('--duration', type=int, default=120, required=False, help="duration") -parser.add_argument('--writers', type=int, default=3, required=False, help="writers per kv") -parser.add_argument('--readers', type=int, default=3, required=False, help="readers per writer per kv") +parser.add_argument('--cmd-log', + default="cmd.log", + required=False, + help="command log") +parser.add_argument('--latency-log', + default="latency.log", + required=False, + help="latency log") +parser.add_argument('--stat-log', + default="stat.log", + required=False, + help="stat log") +parser.add_argument('--duration', + type=int, + default=120, + required=False, + help="duration") +parser.add_argument('--writers', + type=int, + default=3, + required=False, + help="writers per kv") +parser.add_argument('--readers', + type=int, + default=3, + required=False, + help="readers per writer per kv") parser.add_argument('--ss-metric', action='append', required=False) args = parser.parse_args() if args.workload == "mrsw": - asyncio.run(mrsw_aio(args.kv, args.writers, args.readers, args.duration, args.ss_metric, args.cmd_log, args.latency_log, args.stat_log)) + asyncio.run( + mrsw_aio(args.kv, args.writers, args.readers, args.duration, + args.ss_metric, args.cmd_log, args.latency_log, + args.stat_log)) elif args.workload == "comrmw": - asyncio.run(comrmw_aio(args.kv, args.writers, args.readers, args.duration, args.ss_metric, args.cmd_log, args.latency_log, args.stat_log)) \ No newline at end of file + asyncio.run( + comrmw_aio(args.kv, args.writers, args.readers, args.duration, + args.ss_metric, args.cmd_log, args.latency_log, + args.stat_log)) diff --git a/taskfiles/dev.yml b/taskfiles/dev.yml index 421e750d356f6..1fd0b3ed1d269 100644 --- a/taskfiles/dev.yml +++ b/taskfiles/dev.yml @@ -23,6 +23,18 @@ tasks: - test -f "{{.NODE_ROOT}}/bin/node" - test -f "{{.NODE_ROOT}}/bin/npm" + install-yapf: + desc: install yapf + vars: + VENV: '{{.BUILD_ROOT}}/venv/yapf' + cmds: + - python3 -mvenv '{{.VENV}}' + - | + source '{{.VENV}}/bin/activate' + pip3 install --upgrade pip + pip3 install yapf + status: + - test -f '{{.VENV}}/bin/yapf' install-docker-compose: desc: install docker-compose diff --git a/taskfiles/lint.yml b/taskfiles/lint.yml index a6e6111e11800..0ceabe6e2a282 100644 --- a/taskfiles/lint.yml +++ b/taskfiles/lint.yml @@ -8,3 +8,10 @@ tasks: PATH="{{.LLVM_INSTALL_PATH}}/bin":$PATH cd $(git rev-parse --show-toplevel) git ls-files "*.cc" "*.java" "*.h" | xargs -n1 clang-format -i -style=file -fallback-style=none + + python: + desc: lint python code using yapf + deps: + - :dev:install-yapf + cmds: + - '"{{.BUILD_ROOT}}/venv/yapf/bin/yapf" -i -r -e "{{.BUILD_ROOT}}/*" "{{.SRC_DIR}}"'