From 9d6c6c5f541d748dfd7fb082ca5070809cea804a Mon Sep 17 00:00:00 2001 From: Ahmed Date: Mon, 11 Nov 2024 15:24:37 -0500 Subject: [PATCH 1/3] Fix bug caused by zombie Python child process at measurement time. --- scripts/performance/benchmark | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/performance/benchmark b/scripts/performance/benchmark index d1aa8d7a..ce90320f 100755 --- a/scripts/performance/benchmark +++ b/scripts/performance/benchmark @@ -79,9 +79,9 @@ def run_benchmark(pid, output_file, data_interval): memory_used = process_to_measure.memory_info().rss cpu_percent = process_to_measure.cpu_percent() current_net = psutil.net_io_counters(pernic=True)[INTERFACE] - except psutil.AccessDenied: - # Trying to get process information from a closed process will - # result in AccessDenied. + except (psutil.AccessDenied, psutil.ZombieProcess): + # Trying to get process information from a closed or zombie process will + # result in corresponding exceptions. break # Collect data on the in/out network io. From be95ce82dcbb80cbb0e6192c538ab39aa4413b77 Mon Sep 17 00:00:00 2001 From: Ahmed Date: Tue, 31 Dec 2024 15:29:21 -0500 Subject: [PATCH 2/3] Update summarize script to include metrics of each iteration. --- scripts/performance/summarize | 137 ++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 46 deletions(-) diff --git a/scripts/performance/summarize b/scripts/performance/summarize index dd3abc33..ffe22857 100755 --- a/scripts/performance/summarize +++ b/scripts/performance/summarize @@ -12,19 +12,19 @@ Run this script with:: And that should output:: - +------------------------+----------+----------------------+ - | Metric over 1 run(s) | Mean | Standard Deviation | - +========================+==========+======================+ - | Total Time (seconds) | 1.200 | 0.0 | - +------------------------+----------+----------------------+ - | Maximum Memory | 42.3 MiB | 0 Bytes | - +------------------------+----------+----------------------+ - | Maximum CPU (percent) | 88.1 | 0.0 | - +------------------------+----------+----------------------+ - | Average Memory | 33.9 MiB | 0 Bytes | - +------------------------+----------+----------------------+ - | Average CPU (percent) | 30.5 | 0.0 | - +------------------------+----------+----------------------+ ++------------------------+---------------------+-----------+----------------------+ +| Metric over 1 run(s) | Run 1 | Mean | Standard Deviation | ++========================+=====================+===========+======================+ +| Total Time (seconds) | 263.8085448741913 | 263.809 | 0.0 | ++------------------------+---------------------+-----------+----------------------+ +| Maximum Memory | 117.9 MiB | 117.9 MiB | 0 Bytes | ++------------------------+---------------------+-----------+----------------------+ +| Maximum CPU (percent) | 0.2 | 0.2 | 0.0 | ++------------------------+---------------------+-----------+----------------------+ +| Average Memory | 117.5 MiB | 117.5 MiB | 0 Bytes | ++------------------------+---------------------+-----------+----------------------+ +| Average CPU (percent) | 0.07325581395348836 | 0.1 | 0.0 | ++------------------------+---------------------+-----------+----------------------+ The script can also be ran with multiple files: @@ -33,34 +33,56 @@ The script can also be ran with multiple files: And will have a similar output: - +------------------------+----------+----------------------+ - | Metric over 2 run(s) | Mean | Standard Deviation | - +========================+==========+======================+ - | Total Time (seconds) | 1.155 | 0.0449999570847 | - +------------------------+----------+----------------------+ - | Maximum Memory | 42.5 MiB | 110.0 KiB | - +------------------------+----------+----------------------+ - | Maximum CPU (percent) | 94.5 | 6.45 | - +------------------------+----------+----------------------+ - | Average Memory | 35.6 MiB | 1.7 MiB | - +------------------------+----------+----------------------+ - | Average CPU (percent) | 27.5 | 3.03068181818 | - +------------------------+----------+----------------------+ ++------------------------+---------------------+---------------------+-----------+----------------------+ +| Metric over 2 run(s) | Run 1 | Run 2 | Mean | Standard Deviation | ++========================+=====================+=====================+===========+======================+ +| Total Time (seconds) | 263.8085448741913 | 198.05210328102112 | 230.930 | 32.87822079658508 | ++------------------------+---------------------+---------------------+-----------+----------------------+ +| Maximum Memory | 117.9 MiB | 112.4 MiB | 115.2 MiB | 2.7 MiB | ++------------------------+---------------------+---------------------+-----------+----------------------+ +| Maximum CPU (percent) | 0.2 | 0.2 | 0.2 | 0.0 | ++------------------------+---------------------+---------------------+-----------+----------------------+ +| Average Memory | 117.5 MiB | 111.0 MiB | 114.2 MiB | 3.2 MiB | ++------------------------+---------------------+---------------------+-----------+----------------------+ +| Average CPU (percent) | 0.07325581395348836 | 0.09432989690721647 | 0.1 | 0.010537041476864052 | ++------------------------+---------------------+---------------------+-----------+----------------------+ You can also specify the ``--output-format json`` option to print the summary as JSON instead of a pretty printed table:: +{ + "executions": [ { - "total_time": 72.76999998092651, - "std_dev_average_memory": 0.0, - "std_dev_total_time": 0.0, - "average_memory": 56884518.57534247, - "std_dev_average_cpu": 0.0, - "std_dev_max_memory": 0.0, - "average_cpu": 61.19315068493151, - "max_memory": 58331136.0 + "execution_time": 263.8085448741913, + "average_memory": 123170974.75968993, + "max_memory": 123600896.0, + "average_cpu": 0.07325581395348836, + "max_cpu": 0.2, + "end_time": 1735586030.26786 + }, + { + "execution_time": 198.05210328102112, + "average_memory": 116410262.43298969, + "max_memory": 117899264.0, + "average_cpu": 0.09432989690721647, + "max_cpu": 0.2, + "end_time": 1735586230.422445 } + ], + "aggregate_stats": { + "execution_time": 230.9303240776062, + "std_dev_execution_time": 32.87822079658508, + "average_memory": 119790618.5963398, + "std_dev_average_memory": 3380356.16335012, + "max_memory": 120750080.0, + "std_dev_max_memory": 2850816.0, + "average_cpu": 0.08379285543035242, + "std_dev_average_cpu": 0.010537041476864052, + "max_cpu": 0.2, + "std_dev_max_cpu": 0.0 + } +} """ @@ -96,7 +118,7 @@ class Summarizer: self.total_files = 0 self._num_rows = 0 self._start_time = None - self._end_time = None + self._end_times = [] self._totals = { 'time': [], 'average_memory': [], @@ -168,22 +190,30 @@ class Summarizer: table = [ [ 'Total Time (seconds)', + *[f'{self._totals['time'][file]}' for file in range(0, self.total_files)], f'{self.total_time:.3f}', self.std_dev_total_time, ], - ['Maximum Memory', h(self.max_memory), h(self.std_dev_max_memory)], + [ + 'Maximum Memory', + *[f'{h(self._totals['max_memory'][file])}' for file in range(0, self.total_files)], + h(self.max_memory), + h(self.std_dev_max_memory)], [ 'Maximum CPU (percent)', + *[f'{self._totals['max_cpu'][file]}' for file in range(0, self.total_files)], f'{self.max_cpu:.1f}', self.std_dev_max_cpu, ], [ 'Average Memory', + *[f'{h(self._totals['average_memory'][file])}' for file in range(0, self.total_files)], h(self.average_memory), h(self.std_dev_average_memory), ], [ 'Average CPU (percent)', + *[f'{self._totals['average_cpu'][file]}' for file in range(0, self.total_files)], f'{self.average_cpu:.1f}', self.std_dev_average_cpu, ], @@ -192,6 +222,7 @@ class Summarizer: table, headers=[ f'Metric over {self.total_files} run(s)', + *[f'Run {n}' for n in range(1, self.total_files+1)], 'Mean', 'Standard Deviation', ], @@ -205,14 +236,28 @@ class Summarizer: """ return json.dumps( { - 'total_time': self.total_time, - 'std_dev_total_time': self.std_dev_total_time, - 'max_memory': self.max_memory, - 'std_dev_max_memory': self.std_dev_max_memory, - 'average_memory': self.average_memory, - 'std_dev_average_memory': self.std_dev_average_memory, - 'average_cpu': self.average_cpu, - 'std_dev_average_cpu': self.std_dev_average_cpu, + 'executions': [ + { + 'execution_time': self._totals['time'][file], + 'average_memory': self._totals['average_memory'][file], + 'max_memory': self._totals['max_memory'][file], + 'average_cpu': self._totals['average_cpu'][file], + 'max_cpu': self._totals['max_cpu'][file], + 'end_time': self._end_times[file], + } for file in range(self.total_files) + ], + 'aggregate_stats': { + 'execution_time': self.total_time, + 'std_dev_execution_time': self.std_dev_total_time, + 'average_memory': self.average_memory, + 'std_dev_average_memory': self.std_dev_average_memory, + 'max_memory': self.max_memory, + 'std_dev_max_memory': self.std_dev_max_memory, + 'average_cpu': self.average_cpu, + 'std_dev_average_cpu': self.std_dev_average_cpu, + 'max_cpu': self.max_cpu, + 'std_dev_max_cpu': self.std_dev_max_cpu, + } }, indent=2, ) @@ -232,7 +277,7 @@ class Summarizer: self._validate_row(row, benchmark_file) self.process_data_row(row) self._validate_row(row, benchmark_file) - self._end_time = self._get_time(row) + self._end_times.append(self._get_time(row)) self._finalize_processed_data_for_file() def _validate_row(self, row, filename): @@ -261,7 +306,7 @@ class Summarizer: def _finalize_processed_data_for_file(self): # Add numbers to the total, which keeps track of data over # all files provided. - self._totals['time'].append(self._end_time - self._start_time) + self._totals['time'].append(self._end_times[-1] - self._start_time) self._totals['max_cpu'].append(self._maximums['cpu']) self._totals['max_memory'].append(self._maximums['memory']) self._totals['average_cpu'].append( From 5e0446c5d942c3f6c09b203e673d05c91a3ff794 Mon Sep 17 00:00:00 2001 From: Ahmed Date: Thu, 2 Jan 2025 09:39:52 -0500 Subject: [PATCH 3/3] Ruff formatting --- scripts/performance/summarize | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/scripts/performance/summarize b/scripts/performance/summarize index ffe22857..25cd78a8 100755 --- a/scripts/performance/summarize +++ b/scripts/performance/summarize @@ -190,30 +190,46 @@ class Summarizer: table = [ [ 'Total Time (seconds)', - *[f'{self._totals['time'][file]}' for file in range(0, self.total_files)], + *[ + f'{self._totals['time'][file]}' + for file in range(0, self.total_files) + ], f'{self.total_time:.3f}', self.std_dev_total_time, ], [ 'Maximum Memory', - *[f'{h(self._totals['max_memory'][file])}' for file in range(0, self.total_files)], + *[ + f'{h(self._totals['max_memory'][file])}' + for file in range(0, self.total_files) + ], h(self.max_memory), - h(self.std_dev_max_memory)], + h(self.std_dev_max_memory), + ], [ 'Maximum CPU (percent)', - *[f'{self._totals['max_cpu'][file]}' for file in range(0, self.total_files)], + *[ + f'{self._totals['max_cpu'][file]}' + for file in range(0, self.total_files) + ], f'{self.max_cpu:.1f}', self.std_dev_max_cpu, ], [ 'Average Memory', - *[f'{h(self._totals['average_memory'][file])}' for file in range(0, self.total_files)], + *[ + f'{h(self._totals['average_memory'][file])}' + for file in range(0, self.total_files) + ], h(self.average_memory), h(self.std_dev_average_memory), ], [ 'Average CPU (percent)', - *[f'{self._totals['average_cpu'][file]}' for file in range(0, self.total_files)], + *[ + f'{self._totals['average_cpu'][file]}' + for file in range(0, self.total_files) + ], f'{self.average_cpu:.1f}', self.std_dev_average_cpu, ], @@ -222,7 +238,7 @@ class Summarizer: table, headers=[ f'Metric over {self.total_files} run(s)', - *[f'Run {n}' for n in range(1, self.total_files+1)], + *[f'Run {n}' for n in range(1, self.total_files + 1)], 'Mean', 'Standard Deviation', ], @@ -244,7 +260,8 @@ class Summarizer: 'average_cpu': self._totals['average_cpu'][file], 'max_cpu': self._totals['max_cpu'][file], 'end_time': self._end_times[file], - } for file in range(self.total_files) + } + for file in range(self.total_files) ], 'aggregate_stats': { 'execution_time': self.total_time, @@ -257,7 +274,7 @@ class Summarizer: 'std_dev_average_cpu': self.std_dev_average_cpu, 'max_cpu': self.max_cpu, 'std_dev_max_cpu': self.std_dev_max_cpu, - } + }, }, indent=2, )