From 7c949ac1157f4450db38b577b1f6a60adcaf40e8 Mon Sep 17 00:00:00 2001 From: Daniel Biales Date: Sat, 19 Sep 2020 23:06:28 -0400 Subject: [PATCH 1/5] Fix a bug where the latency metrics were getting converted to int. --- script/testing/oltpbench/reporting/parsers/summary_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/testing/oltpbench/reporting/parsers/summary_parser.py b/script/testing/oltpbench/reporting/parsers/summary_parser.py index fb7521f516..89c8d13399 100644 --- a/script/testing/oltpbench/reporting/parsers/summary_parser.py +++ b/script/testing/oltpbench/reporting/parsers/summary_parser.py @@ -70,5 +70,5 @@ def parse_latency_data(latency_dict): latency = {} for key, pattern in LATENCY_ATTRIBUTE_MAPPING: value = get_value_by_pattern(latency_dict, pattern, None) - latency[key] = int(value) if value else value + latency[key] = float(value) if value else value return latency \ No newline at end of file From b3c799b4993596d200260fe1accaeb1820e60ae9 Mon Sep 17 00:00:00 2001 From: Daniel Biales Date: Sat, 19 Sep 2020 23:45:56 -0400 Subject: [PATCH 2/5] Attempted refactor of incremetal latency metrics. Testing by sending data to test DB --- Jenkinsfile-nightly | 8 ++-- .../oltpbench/reporting/parsers/res_parser.py | 43 ++++++++++++------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/Jenkinsfile-nightly b/Jenkinsfile-nightly index 61ed4c2b60..0879283848 100644 --- a/Jenkinsfile-nightly +++ b/Jenkinsfile-nightly @@ -4,7 +4,7 @@ pipeline { environment{ //Do not change. //Performance Storage Service(Django) authentication information. The credentials can only be changed on Jenkins webpage - PSS_CREATOR= credentials('pss-creator') + PSS_CREATOR= credentials('pss-creator-testing') } options { buildDiscarder(logRotator(daysToKeepStr: '30')) @@ -23,9 +23,9 @@ pipeline { sh 'echo y | sudo ./script/installation/packages.sh all' sh 'mkdir build' sh 'cd build && cmake -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DTERRIER_USE_ASAN=OFF -DTERRIER_USE_JEMALLOC=ON -DTERRIER_BUILD_TESTS=OFF .. && make -j$(nproc) terrier' - sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly.json --build-type=release --publish-results=prod --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" - sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_ramdisk.json --build-type=release --publish-results=prod --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" - sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_wal_disabled.json --build-type=release --publish-results=prod --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" + sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly.json --build-type=release --publish-results=test --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" + sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_ramdisk.json --build-type=release --publish-results=test --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" + sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_wal_disabled.json --build-type=release --publish-results=test --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" archiveArtifacts(artifacts: 'build/oltp_result/**/*.*', excludes: 'build/oltp_result/**/*.csv', fingerprint: true) } diff --git a/script/testing/oltpbench/reporting/parsers/res_parser.py b/script/testing/oltpbench/reporting/parsers/res_parser.py index be6050331c..03c26b3982 100644 --- a/script/testing/oltpbench/reporting/parsers/res_parser.py +++ b/script/testing/oltpbench/reporting/parsers/res_parser.py @@ -1,6 +1,8 @@ import csv import json +from oltpbench.reporting.utils import get_value_by_pattern +from oltpbench.reporting.constants import LATENCY_ATTRIBUTE_MAPPING def parse_res_file(path): """Read data from file ends with ".res". @@ -12,22 +14,33 @@ def parse_res_file(path): incremental_metrics (list, json array): The throughput at different time. """ - time, throughput, min_lat, lat_25th, median_lat, avg_lat, lat_75th, lat_90th, lat_95th, lat_99th, max_lat = [ - ], [], [], [], [], [], [], [], [], [], [] + # time, throughput, min_lat, lat_25th, median_lat, avg_lat, lat_75th, lat_90th, lat_95th, lat_99th, max_lat = [ + # ], [], [], [], [], [], [], [], [], [], [] with open(path) as csvfile: reader = csv.DictReader(csvfile, delimiter=',') + incremental_metrics = [] for row in reader: - time.append(float(row['time(sec)'])) - throughput.append(float(row[' throughput(req/sec)'])) - min_lat.append(float(row[' min_lat(ms)'])) - lat_25th.append(float(row[' 25th_lat(ms)'])) - median_lat.append(float(row[' median_lat(ms)'])) - avg_lat.append(float(row[' avg_lat(ms)'])) - lat_75th.append(float(row[' 75th_lat(ms)'])) - lat_90th.append(float(row[' 90th_lat(ms)'])) - lat_95th.append(float(row[' 95th_lat(ms)'])) - lat_99th.append(float(row[' 99th_lat(ms)'])) - max_lat.append(float(row[' max_lat(ms)'])) - incremental_metrics = [{"time": t, "throughput": tp, "latency":{"min": ml, "l_25": l25, "median": mel, "avg": al, "l_75": l75, "l_90": l90, "l_95": l95, "l_99": l99, "max": mal}} - for t, tp, ml, l25, mel, al, l75, l90, l95, l99, mal in zip(time, throughput, min_lat, lat_25th, median_lat, avg_lat, lat_75th, lat_90th, lat_95th, lat_99th, max_lat)] + metrics_instance = { + "time": float(get_value_by_pattern(row,'time',None)), + "throughput": float(get_value_by_pattern(row, 'throughput', None)) + } + # time.append(float(row['time(sec)'])) + # throughput.append(float(row[' throughput(req/sec)'])) + latency = {} + for key, pattern in LATENCY_ATTRIBUTE_MAPPING: + value = get_value_by_pattern(row, pattern, None) + latency[key] = float(value) if value else value + metrics_instance['latency'] = latency + incremental_metrics.append(metrics_instance) + # min_lat.append(float(row[' min_lat(ms)'])) + # lat_25th.append(float(row[' 25th_lat(ms)'])) + # median_lat.append(float(row[' median_lat(ms)'])) + # avg_lat.append(float(row[' avg_lat(ms)'])) + # lat_75th.append(float(row[' 75th_lat(ms)'])) + # lat_90th.append(float(row[' 90th_lat(ms)'])) + # lat_95th.append(float(row[' 95th_lat(ms)'])) + # lat_99th.append(float(row[' 99th_lat(ms)'])) + # max_lat.append(float(row[' max_lat(ms)'])) + # incremental_metrics = [{"time": t, "throughput": tp, "latency":latency} + # for t, tp, ml, l25, mel, al, l75, l90, l95, l99, mal in zip(time, throughput, min_lat, lat_25th, median_lat, avg_lat, lat_75th, lat_90th, lat_95th, lat_99th, max_lat)] return incremental_metrics From 5fb5dd940287f89c4b9eff6a6c5cec04687b1820 Mon Sep 17 00:00:00 2001 From: Daniel Biales Date: Sun, 20 Sep 2020 13:47:22 -0400 Subject: [PATCH 3/5] Refactor res_parser to simplify the code and for more reuse --- script/testing/oltpbench/reporting/constants.py | 2 +- .../oltpbench/reporting/parsers/res_parser.py | 17 +---------------- .../reporting/parsers/summary_parser.py | 2 +- script/testing/oltpbench/test_case_oltp.py | 4 ++-- 4 files changed, 5 insertions(+), 20 deletions(-) diff --git a/script/testing/oltpbench/reporting/constants.py b/script/testing/oltpbench/reporting/constants.py index 1e5e252583..38dcb489da 100644 --- a/script/testing/oltpbench/reporting/constants.py +++ b/script/testing/oltpbench/reporting/constants.py @@ -3,4 +3,4 @@ UNKNOWN_RESULT = 'unknown' LATENCY_ATTRIBUTE_MAPPING = [ ('l_25','25'),('l_75','75'),('l_90','90'), ('l_95','95'), ('l_99','99'), - ('avg','average'),('median','median'),('min','minimum'), ('max','maximum')] \ No newline at end of file + ('avg','av'),('median','median'),('min','min'), ('max','max')] \ No newline at end of file diff --git a/script/testing/oltpbench/reporting/parsers/res_parser.py b/script/testing/oltpbench/reporting/parsers/res_parser.py index 03c26b3982..a37c54c8aa 100644 --- a/script/testing/oltpbench/reporting/parsers/res_parser.py +++ b/script/testing/oltpbench/reporting/parsers/res_parser.py @@ -14,8 +14,6 @@ def parse_res_file(path): incremental_metrics (list, json array): The throughput at different time. """ - # time, throughput, min_lat, lat_25th, median_lat, avg_lat, lat_75th, lat_90th, lat_95th, lat_99th, max_lat = [ - # ], [], [], [], [], [], [], [], [], [], [] with open(path) as csvfile: reader = csv.DictReader(csvfile, delimiter=',') incremental_metrics = [] @@ -24,23 +22,10 @@ def parse_res_file(path): "time": float(get_value_by_pattern(row,'time',None)), "throughput": float(get_value_by_pattern(row, 'throughput', None)) } - # time.append(float(row['time(sec)'])) - # throughput.append(float(row[' throughput(req/sec)'])) latency = {} for key, pattern in LATENCY_ATTRIBUTE_MAPPING: value = get_value_by_pattern(row, pattern, None) - latency[key] = float(value) if value else value + latency[key] = float("{:.4}".format(value)) if value else value metrics_instance['latency'] = latency incremental_metrics.append(metrics_instance) - # min_lat.append(float(row[' min_lat(ms)'])) - # lat_25th.append(float(row[' 25th_lat(ms)'])) - # median_lat.append(float(row[' median_lat(ms)'])) - # avg_lat.append(float(row[' avg_lat(ms)'])) - # lat_75th.append(float(row[' 75th_lat(ms)'])) - # lat_90th.append(float(row[' 90th_lat(ms)'])) - # lat_95th.append(float(row[' 95th_lat(ms)'])) - # lat_99th.append(float(row[' 99th_lat(ms)'])) - # max_lat.append(float(row[' max_lat(ms)'])) - # incremental_metrics = [{"time": t, "throughput": tp, "latency":latency} - # for t, tp, ml, l25, mel, al, l75, l90, l95, l99, mal in zip(time, throughput, min_lat, lat_25th, median_lat, avg_lat, lat_75th, lat_90th, lat_95th, lat_99th, max_lat)] return incremental_metrics diff --git a/script/testing/oltpbench/reporting/parsers/summary_parser.py b/script/testing/oltpbench/reporting/parsers/summary_parser.py index 89c8d13399..031fbf4e50 100644 --- a/script/testing/oltpbench/reporting/parsers/summary_parser.py +++ b/script/testing/oltpbench/reporting/parsers/summary_parser.py @@ -70,5 +70,5 @@ def parse_latency_data(latency_dict): latency = {} for key, pattern in LATENCY_ATTRIBUTE_MAPPING: value = get_value_by_pattern(latency_dict, pattern, None) - latency[key] = float(value) if value else value + latency[key] = float("{:.4}".format(value)) if value else value return latency \ No newline at end of file diff --git a/script/testing/oltpbench/test_case_oltp.py b/script/testing/oltpbench/test_case_oltp.py index 19742708f3..c7e85ee9bd 100644 --- a/script/testing/oltpbench/test_case_oltp.py +++ b/script/testing/oltpbench/test_case_oltp.py @@ -119,8 +119,8 @@ def run_post_test(self): # publish results if self.publish_results: - report(self.publish_results, self.server_data,os.path.join( - os.getcwd(), "oltp_result",self.filename_suffix),self.publish_username,self.publish_password,self.query_mode) + report(self.publish_results, self.server_data, os.path.join( + os.getcwd(), "oltp_result",self.filename_suffix), self.publish_username, self.publish_password, self.query_mode) def create_result_dir(self): if not os.path.exists(self.test_result_dir): From 56797d29aeb3bfc1e68a6186c529e4b6e146342c Mon Sep 17 00:00:00 2001 From: Daniel Biales Date: Sun, 20 Sep 2020 17:12:05 -0400 Subject: [PATCH 4/5] revert code to send performance results to prod, since the tests already passed --- Jenkinsfile-nightly | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile-nightly b/Jenkinsfile-nightly index 0879283848..9136a3b8d3 100644 --- a/Jenkinsfile-nightly +++ b/Jenkinsfile-nightly @@ -23,9 +23,9 @@ pipeline { sh 'echo y | sudo ./script/installation/packages.sh all' sh 'mkdir build' sh 'cd build && cmake -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Release -DTERRIER_USE_ASAN=OFF -DTERRIER_USE_JEMALLOC=ON -DTERRIER_BUILD_TESTS=OFF .. && make -j$(nproc) terrier' - sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly.json --build-type=release --publish-results=test --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" - sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_ramdisk.json --build-type=release --publish-results=test --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" - sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_wal_disabled.json --build-type=release --publish-results=test --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" + sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly.json --build-type=release --publish-results=prod --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" + sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_ramdisk.json --build-type=release --publish-results=prod --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" + sh "cd build && timeout 3h python3 ../script/testing/oltpbench/run_oltpbench.py --config-file=../script/testing/oltpbench/configs/nightly/nightly_wal_disabled.json --build-type=release --publish-results=prod --publish-username=${PSS_CREATOR_USR} --publish-password=${PSS_CREATOR_PSW}" archiveArtifacts(artifacts: 'build/oltp_result/**/*.*', excludes: 'build/oltp_result/**/*.csv', fingerprint: true) } From 6c9656f83a642f6354aa94cf6d66606060da5480 Mon Sep 17 00:00:00 2001 From: Daniel Biales Date: Sun, 20 Sep 2020 17:13:31 -0400 Subject: [PATCH 5/5] reset PSS credentials in Jenkins --- Jenkinsfile-nightly | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile-nightly b/Jenkinsfile-nightly index 9136a3b8d3..61ed4c2b60 100644 --- a/Jenkinsfile-nightly +++ b/Jenkinsfile-nightly @@ -4,7 +4,7 @@ pipeline { environment{ //Do not change. //Performance Storage Service(Django) authentication information. The credentials can only be changed on Jenkins webpage - PSS_CREATOR= credentials('pss-creator-testing') + PSS_CREATOR= credentials('pss-creator') } options { buildDiscarder(logRotator(daysToKeepStr: '30'))