From 2551c4b2cacd7771a819c3911b51f31eaa055545 Mon Sep 17 00:00:00 2001 From: Desiki-high Date: Sat, 22 Apr 2023 17:05:33 +0800 Subject: [PATCH] misc: add read-amount and read-count for benchmark We should add the read-amount and read-count for nydus benchmark to compare nydus and zran or different batchsize. Signed-off-by: Desiki-high --- misc/benchmark/bench.py | 12 +++++++---- misc/benchmark/benchmark.py | 6 +++--- misc/benchmark/benchmark_summary.sh | 31 ++++++++++++++++------------ misc/benchmark/metrics.py | 32 +++++++++++++++++++++++++++-- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/misc/benchmark/bench.py b/misc/benchmark/bench.py index f6f2f15e148..94bf9e1f4d1 100644 --- a/misc/benchmark/bench.py +++ b/misc/benchmark/bench.py @@ -15,6 +15,8 @@ from datetime import datetime from io import TextIOWrapper +import metrics + def logging_setup(logging_stream=sys.stderr): root = logging.getLogger() @@ -182,11 +184,13 @@ def run_cmd_url_wait(self, repo, runargs): run_elapsed = datetime.timestamp(end_run) - datetime.timestamp(start_run) print("Run time: %f s" % run_elapsed) - + read_amount, read_count = "-", "-" + if self.snapshotter == "nydus": + read_amount, read_count = metrics.collect_backend() if self.cleanup: self.clean_up(image_ref, container_id) - return pull_elapsed, create_elapsed, run_elapsed + return pull_elapsed, create_elapsed, run_elapsed, read_amount, read_count def pull_cmd(self, image_ref): insecure_flag = "--insecure-registry" if self.insecure_registry else "" @@ -249,11 +253,11 @@ def bench_image(local_registry, insecure_local_registry, image, f: TextIOWrapper cleanup=True, insecure_registry=insecure_local_registry, ) - pull_elapsed, create_elapsed, run_elapsed = runner.run(bench) + pull_elapsed, create_elapsed, run_elapsed, read_amount, read_count = runner.run(bench) total_elapsed = f"{pull_elapsed + create_elapsed + run_elapsed: .6f}" pull_elapsed = f"{pull_elapsed: .6f}" create_elapsed = f"{create_elapsed: .6f}" run_elapsed = f"{run_elapsed: .6f}" - line = f"{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed}" + line = f"{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed},{read_amount},{read_count}" f.writelines(line + "\n") f.flush() diff --git a/misc/benchmark/benchmark.py b/misc/benchmark/benchmark.py index 9ab9cff4d32..080fbde06bb 100644 --- a/misc/benchmark/benchmark.py +++ b/misc/benchmark/benchmark.py @@ -43,9 +43,9 @@ def main(): def collect_metrics(cfg: dict, image: str) -> str: """ - collect metrics + collect container access metrics """ - return metrics.collect(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image)) + return metrics.collect_access(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image)) def start_bench(cfg: dict, image: str, mode: str): @@ -53,7 +53,7 @@ def start_bench(cfg: dict, image: str, mode: str): bench oci, nydus without prefetch, nydus with all prefetch, nydus witch prefetch file list """ f = open(util.image_repo(image) + ".csv", "w") - csv_headers = "repo,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s)" + csv_headers = "repo,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s),read_amount(MB),read_count" f.writelines(csv_headers + "\n") f.flush() if mode == "oci": diff --git a/misc/benchmark/benchmark_summary.sh b/misc/benchmark/benchmark_summary.sh index 7293ab7036d..068d838e218 100644 --- a/misc/benchmark/benchmark_summary.sh +++ b/misc/benchmark/benchmark_summary.sh @@ -7,21 +7,26 @@ sudo install -m 755 benchmark-nydus-no-prefetch/wordpress.csv nydus-no-prefetch. sudo install -m 755 benchmark-nydus-all-prefetch/wordpress.csv nydus-all-prefetch.csv sudo install -m 755 benchmark-nydus-filelist-prefetch/wordpress.csv nydus-filelist-prefetch.csv -echo "| benchmark-result | pull-elapsed(s) | create-elapsed(s) | run-elapsed(s) | total-elapsed(s) |" -echo "|:-------|:-----------------:|:-------------------:|:----------------:|:------------------:|" +echo "| bench-result | pull-elapsed(s) | create-elapsed(s) | run-elapsed(s) | total-elapsed(s) |read-amount(MB) |read-count |" +echo "|:-------------|:---------------:|:-----------------:|:--------------:|:----------------:|:--------------:|:---------:|" files=(oci.csv nydus-all-prefetch.csv zran-all-prefetch.csv nydus-no-prefetch.csv zran-no-prefetch.csv nydus-filelist-prefetch.csv) for file in "${files[@]}"; do -if ! [ -f "$file" ]; then - continue -fi -filename=$(basename "$file" .csv) -tail -n +2 "$file" | while read line; do - pull=$(echo "$line" | cut -d ',' -f 2) - create=$(echo "$line" | cut -d ',' -f 3) - run=$(echo "$line" | cut -d ',' -f 4) - total=$(echo "$line" | cut -d ',' -f 5) - printf "| %s | %s | %s | %s | %s |\n" "$filename" "$pull" "$create" "$run" "$total" -done + if ! [ -f "$file" ]; then + continue + fi + filename=$(basename "$file" .csv) + tail -n +2 "$file" | while read line; do + if [ -z "$line" ]; then + continue + fi + pull=$(echo "$line" | cut -d ',' -f 2) + create=$(echo "$line" | cut -d ',' -f 3) + run=$(echo "$line" | cut -d ',' -f 4) + total=$(echo "$line" | cut -d ',' -f 5) + amount=$(echo "$line" | cut -d ',' -f 6) + count=$(echo "$line" | cut -d ',' -f 7) + printf "| %s | %s | %s | %s | %s | %s | %s |\n" "$filename" "$pull" "$create" "$run" "$total" "$amount" "$count" + done done diff --git a/misc/benchmark/metrics.py b/misc/benchmark/metrics.py index 4f1b3c1fe4a..763166a80db 100755 --- a/misc/benchmark/metrics.py +++ b/misc/benchmark/metrics.py @@ -10,6 +10,7 @@ import subprocess import time import urllib.request +from typing import Tuple """ define some file path, and sock api url @@ -18,6 +19,7 @@ TEMP_DIR = "tmp" URL_PREFIX = "http://localhost/api" ACCESS_PATTERN_METRICS = "/v1/metrics/pattern" +BACKEND_METRICS = "/v1/metrics/backend" BOOTSTRAP_DIR = "/var/lib/containerd-nydus/snapshots" API_DIR = "/var/lib/containerd-nydus/socket" @@ -189,6 +191,16 @@ def get_access_pattern(sock, bootstap_data): return access_pattern_list +def get_backend_metrics(sock) -> dict: + """ + get the backend metrics from the sock + """ + with open(send_request(sock, BACKEND_METRICS), 'r') as file: + content = file.read() + + return eval(content) + + def random_string(): """ generate a random string of fixed length @@ -248,12 +260,28 @@ def init(): os.mkdir(TEMP_DIR) -def collect(local_registry, insecure_local_registry, image) -> str: +def collect_access(local_registry, insecure_local_registry, image) -> str: """ - serve for benchmark.py + collect access pattern for benchmark.py """ init() cfg = {"registry": local_registry, "insecure_registry": insecure_local_registry, "image": image} file = MetricsCollector(cfg).start_collet() shutil.rmtree(TEMP_DIR) return file + + +def collect_backend() -> Tuple[str, str]: + """ + collect backend metrics for benchmark.py + + return (Read Amount, Read Count) + """ + init() + socket = search_file(API_DIR, "api.sock") + if socket == None: + print("can't find the api.sock") + exit(1) + backend = get_backend_metrics(socket) + shutil.rmtree(TEMP_DIR) + return round(backend["read_amount_total"] / 1024 / 1024, 2), backend["read_count"]