diff --git a/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py b/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py index eeb089dcd2..debbf0c389 100644 --- a/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py +++ b/perfmetrics/scripts/testing_on_gke/examples/dlio/parse_logs.py @@ -50,7 +50,7 @@ def downloadDlioOutputs(dlioWorkloads): result = subprocess.run( [ "gsutil", - "-m", # download files parallely + "-m", # download multiple files parallelly "-q", # download silently without any logs "cp", "-r", @@ -69,19 +69,27 @@ def downloadDlioOutputs(dlioWorkloads): prog="DLIO Unet3d test output parser", description=( "This program takes in a json test-config file and parses it for" - " output buckets. From each output bucket, it downloads all the dlio" + " output buckets. From each output bucket, it downloads all the DLIO" " output logs from gs:///logs/ localy to" - f" {LOCAL_LOGS_LOCATION} and parses them for dlio test runs and their" + f" {LOCAL_LOGS_LOCATION} and parses them for DLIO test runs and their" " output metrics." ), ) - parser.add_argument("--workload-config") + parser.add_argument( + "--workload-config", + description=( + "A json configuration file to define workloads that were run to" + " generate the outputs that should be parsed." + ), + required=True, + ) parser.add_argument( "--project-number", help=( "project-number (e.g. 93817472919) is needed to fetch the cpu/memory" " utilization data from GCP." ), + required=True, ) args = parser.parse_args() @@ -129,6 +137,7 @@ def downloadDlioOutputs(dlioWorkloads): except: print(f"failed to json-parse {per_epoch_stats_file}") continue + with open(summary_file, "r") as f: try: summary_data = json.load(f) diff --git a/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py b/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py index 269716e8b5..9bf207c3b3 100644 --- a/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py +++ b/perfmetrics/scripts/testing_on_gke/examples/fio/parse_logs.py @@ -58,7 +58,7 @@ def downloadFioOutputs(fioWorkloads): [ "gsutil", "-m", # download multiple files parallelly - "-q", # do it quietly + "-q", # download silently without any logs "cp", "-r", f"gs://{fioWorkload.bucket}/fio-output", @@ -76,19 +76,27 @@ def downloadFioOutputs(fioWorkloads): prog="DLIO Unet3d test output parser", description=( "This program takes in a json test-config file and parses it for" - " output buckets.From each output bucket, it downloads all the dlio" + " output buckets. From each output bucket, it downloads all the FIO" " output logs from gs:///logs/ locally to" - f" {LOCAL_LOGS_LOCATION} and parses them for dlio test runs and their" + f" {LOCAL_LOGS_LOCATION} and parses them for FIO test runs and their" " output metrics." ), ) - parser.add_argument("--workload-config") + parser.add_argument( + "--workload-config", + description=( + "A json configuration file to define workloads that were run to" + " generate the outputs that should be parsed." + ), + required=True, + ) parser.add_argument( "--project-number", help=( - "project-number (93817472919) is needed to fetch the cpu/memory" + "project-number (e.g. 93817472919) is needed to fetch the cpu/memory" " utilization data from GCP." ), + required=True, ) args = parser.parse_args() @@ -125,6 +133,7 @@ def downloadFioOutputs(fioWorkloads): print(f"ignoring file {per_epoch_output} as it's not a json file") continue + gcsfuse_mount_options = "" gcsfuse_mount_options_file = root + "/gcsfuse_mount_options" if os.path.isfile(gcsfuse_mount_options_file): with open(gcsfuse_mount_options_file) as f: