From f04a7ce97812a8069c756e114b02c8dfc1443396 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Thu, 11 Jul 2024 18:21:03 -0300 Subject: [PATCH] Rework beluga_benchmark using LAMBKIN Signed-off-by: Michel Hidalgo --- beluga_benchmark/CMakeLists.txt | 37 +- beluga_benchmark/README.md | 3 +- beluga_benchmark/beluga_benchmark/__init__.py | 23 - .../beluga_benchmark/compare_results.py | 290 ----------- .../beluga_benchmark/exceptions.py | 17 - .../beluga_benchmark/timem_results.py | 100 ---- beluga_benchmark/config/qos_override.yml | 6 + beluga_benchmark/docs/BENCHMARKING.md | 108 +---- .../docs/reports/2023-03-16/REPORT.md | 58 --- .../docs/reports/2023-03-16/params_par.yaml | 93 ---- .../docs/reports/2023-03-16/results-1.png | Bin 107207 -> 0 bytes .../docs/reports/2023-03-16/results-2.png | Bin 96952 -> 0 bytes .../docs/reports/2023-05-20/REPORT.md | 123 ----- .../2023-05-20/beam_par_beluga_vs_amcl.png | Bin 180356 -> 0 bytes .../docs/reports/2023-05-20/beam_params.yaml | 93 ---- .../reports/2023-05-20/beam_params_par.yaml | 93 ---- .../2023-05-20/beam_seq_beluga_vs_amcl.png | Bin 183136 -> 0 bytes .../likelihood_par_beluga_vs_amcl.png | Bin 177190 -> 0 bytes .../2023-05-20/likelihood_params_par.yaml | 93 ---- .../likelihood_seq_beluga_vs_amcl.png | Bin 178835 -> 0 bytes .../docs/reports/2023-06-03/REPORT.md | 144 ------ .../beam_beluga_seq_vs_amcl_log.png | Bin 154231 -> 0 bytes .../beam_beluga_vs_beluga_vs_amcl.png | Bin 248847 -> 0 bytes .../docs/reports/2023-06-03/beam_params.yaml | 93 ---- .../reports/2023-06-03/beam_params_par.yaml | 93 ---- .../likelihood_beluga_seq_vs_amcl_log.png | Bin 196793 -> 0 bytes .../likelihood_beluga_vs_beluga_vs_amcl.png | Bin 245614 -> 0 bytes .../reports/2023-06-03/likelihood_params.yaml | 93 ---- .../2023-06-03/likelihood_params_par.yaml | 93 ---- .../docs/reports/2023-09-02/REPORT.md | 122 ----- .../beam_beluga_vs_beluga_vs_amcl.png | Bin 187416 -> 0 bytes .../beam_beluga_vs_beluga_vs_amcl_linear.png | Bin 140633 -> 0 bytes .../likelihood_beluga_vs_beluga_vs_amcl.png | Bin 189106 -> 0 bytes ...lihood_beluga_vs_beluga_vs_amcl_linear.png | Bin 148939 -> 0 bytes .../baseline_configurations/beam_params.yaml | 93 ---- .../beam_params_par.yaml | 93 ---- .../likelihood_params.yaml | 93 ---- .../likelihood_params_par.yaml | 93 ---- beluga_benchmark/launch/beluga_ndt.launch | 32 ++ beluga_benchmark/launch/beluga_vs_nav2.launch | 38 ++ beluga_benchmark/package.xml | 33 +- .../params.yaml => params/amcl.yaml} | 22 +- .../ndt_mcl.yaml} | 44 +- .../reports/ndt_swept_report/conf.py | 66 +++ .../reports/ndt_swept_report/index.rst | 443 +++++++++++++++++ .../reports/nominal_report/conf.py | 72 +++ .../reports/nominal_report/index.rst | 201 ++++++++ beluga_benchmark/reports/swept_report/conf.py | 66 +++ .../reports/swept_report/index.rst | 454 ++++++++++++++++++ .../scripts/benchmarking/compare_results | 26 - .../scripts/benchmarking/parameterized_run | 134 ------ .../scripts/benchmarking/timem_results | 26 - beluga_benchmark/scripts/ndt_swept.robot | 72 +++ beluga_benchmark/scripts/nominal.robot | 62 +++ beluga_benchmark/scripts/swept.robot | 72 +++ docker/images/humble/Dockerfile | 39 +- docker/images/iron/Dockerfile | 39 +- docker/images/jazzy/Dockerfile | 39 +- docker/images/rolling/Dockerfile | 39 +- docker/run.sh | 2 +- 60 files changed, 1716 insertions(+), 2452 deletions(-) delete mode 100644 beluga_benchmark/beluga_benchmark/__init__.py delete mode 100644 beluga_benchmark/beluga_benchmark/compare_results.py delete mode 100644 beluga_benchmark/beluga_benchmark/exceptions.py delete mode 100644 beluga_benchmark/beluga_benchmark/timem_results.py create mode 100644 beluga_benchmark/config/qos_override.yml delete mode 100644 beluga_benchmark/docs/reports/2023-03-16/REPORT.md delete mode 100644 beluga_benchmark/docs/reports/2023-03-16/params_par.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-03-16/results-1.png delete mode 100644 beluga_benchmark/docs/reports/2023-03-16/results-2.png delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/REPORT.md delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/beam_par_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/beam_params.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/beam_params_par.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/beam_seq_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/likelihood_par_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/likelihood_params_par.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-05-20/likelihood_seq_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/REPORT.md delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/beam_beluga_seq_vs_amcl_log.png delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/beam_beluga_vs_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/beam_params.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/beam_params_par.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/likelihood_beluga_seq_vs_amcl_log.png delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/likelihood_beluga_vs_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/likelihood_params.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-06-03/likelihood_params_par.yaml delete mode 100644 beluga_benchmark/docs/reports/2023-09-02/REPORT.md delete mode 100644 beluga_benchmark/docs/reports/2023-09-02/beam_beluga_vs_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-09-02/beam_beluga_vs_beluga_vs_amcl_linear.png delete mode 100644 beluga_benchmark/docs/reports/2023-09-02/likelihood_beluga_vs_beluga_vs_amcl.png delete mode 100644 beluga_benchmark/docs/reports/2023-09-02/likelihood_beluga_vs_beluga_vs_amcl_linear.png delete mode 100644 beluga_benchmark/docs/reports/baseline_configurations/beam_params.yaml delete mode 100644 beluga_benchmark/docs/reports/baseline_configurations/beam_params_par.yaml delete mode 100644 beluga_benchmark/docs/reports/baseline_configurations/likelihood_params.yaml delete mode 100644 beluga_benchmark/docs/reports/baseline_configurations/likelihood_params_par.yaml create mode 100644 beluga_benchmark/launch/beluga_ndt.launch create mode 100644 beluga_benchmark/launch/beluga_vs_nav2.launch rename beluga_benchmark/{docs/reports/2023-03-16/params.yaml => params/amcl.yaml} (86%) rename beluga_benchmark/{docs/reports/2023-05-20/likelihood_params.yaml => params/ndt_mcl.yaml} (80%) create mode 100644 beluga_benchmark/reports/ndt_swept_report/conf.py create mode 100644 beluga_benchmark/reports/ndt_swept_report/index.rst create mode 100644 beluga_benchmark/reports/nominal_report/conf.py create mode 100644 beluga_benchmark/reports/nominal_report/index.rst create mode 100644 beluga_benchmark/reports/swept_report/conf.py create mode 100644 beluga_benchmark/reports/swept_report/index.rst delete mode 100755 beluga_benchmark/scripts/benchmarking/compare_results delete mode 100755 beluga_benchmark/scripts/benchmarking/parameterized_run delete mode 100755 beluga_benchmark/scripts/benchmarking/timem_results create mode 100755 beluga_benchmark/scripts/ndt_swept.robot create mode 100755 beluga_benchmark/scripts/nominal.robot create mode 100755 beluga_benchmark/scripts/swept.robot diff --git a/beluga_benchmark/CMakeLists.txt b/beluga_benchmark/CMakeLists.txt index 90f1d3f5e..43581f77f 100644 --- a/beluga_benchmark/CMakeLists.txt +++ b/beluga_benchmark/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2023 Ekumen, Inc. +# Copyright 2024 Ekumen, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,30 +15,17 @@ cmake_minimum_required(VERSION 3.16) project(beluga_benchmark) -set(ROS_VERSION $ENV{ROS_VERSION}) -if(NOT ROS_VERSION) - if(CATKIN_DEVEL_PREFIX OR CATKIN_BUILD_BINARY_PACKAGE) - set(ROS_VERSION 1) - else() - find_package(ament_cmake QUIET) - if(ament_cmake_FOUND) - set(ROS_VERSION 2) - endif() - endif() -endif() -if(ROS_VERSION EQUAL 2) - find_package(ament_cmake REQUIRED) - find_package(ament_cmake_python REQUIRED) - find_package(beluga_amcl REQUIRED) +find_package(ament_cmake REQUIRED) +find_package(ament_cmake_python REQUIRED) - ament_python_install_package(${PROJECT_NAME}) +install(PROGRAMS scripts/nominal.robot scripts/swept.robot + scripts/ndt_swept.robot DESTINATION lib/${PROJECT_NAME}) - install(DIRECTORY scripts/profiling scripts/benchmarking - USE_SOURCE_PERMISSIONS DESTINATION lib/${PROJECT_NAME}) +install( + DIRECTORY config + params + launch + reports + DESTINATION share/${PROJECT_NAME}) - ament_package() -elseif(ROS_VERSION EQUAL 1) - install(CODE "message(STATUS 'Nothing to be done')") -else() - message(FATAL_ERROR "Unknown ROS distribution version: ${ROS_VERSION}") -endif() +ament_package() diff --git a/beluga_benchmark/README.md b/beluga_benchmark/README.md index b942fbccd..cde1b24c2 100644 --- a/beluga_benchmark/README.md +++ b/beluga_benchmark/README.md @@ -1,9 +1,8 @@ # Beluga Benchmark -This package contains scripts to benchmark, profile and also compare Beluga AMCL with other implementations. +This package contains scripts to benchmark, profile, and also compare Beluga AMCL with other implementations. ## Getting Started -- View [reports](docs/reports/) comparing Beluga's performance to other AMCL implementations. - [How to benchmark Beluga?](docs/BENCHMARKING.md) - [How to profile Beluga?](docs/PROFILING.md) diff --git a/beluga_benchmark/beluga_benchmark/__init__.py b/beluga_benchmark/beluga_benchmark/__init__.py deleted file mode 100644 index 4be18feaa..000000000 --- a/beluga_benchmark/beluga_benchmark/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2023 Ekumen, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .exceptions import ScriptError -from . import compare_results -from . import timem_results - -__all__ = [ - 'ScriptError', - 'compare_results', - 'timem_results', -] diff --git a/beluga_benchmark/beluga_benchmark/compare_results.py b/beluga_benchmark/beluga_benchmark/compare_results.py deleted file mode 100644 index d522276bd..000000000 --- a/beluga_benchmark/beluga_benchmark/compare_results.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright 2023 Ekumen, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from pathlib import Path -import re - -from evo import main_ape -from evo.tools import file_interface -import matplotlib.pyplot as plt -from rosbag2_py._info import Info as Bag2Info -import pandas as pd -from itertools import cycle - -from . import timem_results -from .exceptions import ScriptError - -DIR_NAME_PATTERN = 'benchmark_([0-9]+)_particles_output' -SUPPORTED_TOPIC_NAMES = ('/pose', '/amcl_pose') -EVO_RESULTS_FILE_NAME = 'evo_results.zip' -TERMINAL_OUTPUT_LOG_FILE_NAME = 'output.log' - - -def get_bag_est_topic(bag_path: Path): - metadata = Bag2Info().read_metadata(str(bag_path), '') - for topic_name in SUPPORTED_TOPIC_NAMES: - if any( - topic_info.topic_metadata.name == topic_name - for topic_info in metadata.topics_with_message_count - ): - return topic_name - raise ScriptError( - f'Estimate pose topic was not found in bag file, expected names: {SUPPORTED_TOPIC_NAMES}' - ) - - -def run_evo_ape(dir_path: Path): - evo_results_path = dir_path / EVO_RESULTS_FILE_NAME - if evo_results_path.exists(): - return - bag_path = dir_path / 'rosbag' - est_topic = get_bag_est_topic(bag_path) - arg_parser = main_ape.parser() - args = arg_parser.parse_args( - [ - 'bag2', - str(bag_path), - '/odometry/ground_truth', - est_topic, - '--save_results', - str(evo_results_path.resolve()), - ] - ) - main_ape.run(args) - - -def read_evo_stats(zip_file_path): - # keys: "rmse", "mean", "median", "std", "min", "max", "sse" - return file_interface.load_res_file(zip_file_path).stats - - -def parse_latency_data(output_file_path): - import re - - # Example line: - pattern = re.compile( - r'\[.*\] \[INFO\] \[([0-9]*\.[0-9]*)\] \[amcl\]: Particle filter update iteration stats: ' - + r'[0-9]* particles [0-9]* points - ([0-9]*\.[0-9]*)ms' - ) - latencies_seq = [] - with open(output_file_path, 'r') as f: - for line in f: - if not pattern.match(line): - continue - # first match is the whole line, second match the timestamp, and third the latency - latency = pattern.match(line).group(2) - latencies_seq.append(float(latency) * 1e-3) # convert to seconds - return ( - { - 'latency_min': min(latencies_seq), - 'latency_max': max(latencies_seq), - 'latency_mean': sum(latencies_seq) / len(latencies_seq), - 'latency_median': sorted(latencies_seq)[len(latencies_seq) // 2], - } - if latencies_seq - else { - 'latency_min': 0.0, - 'latency_max': 0.0, - 'latency_mean': 0.0, - 'latency_median': 0.0, - } - ) - - -def create_parameterized_series(results_path: Path): - particles = [] - peak_rss = [] - cpu_usage = [] - ape_rmse = [] - ape_mean = [] - ape_std = [] - ape_median = [] - ape_max = [] - latency_min = [] - latency_max = [] - latency_mean = [] - latency_median = [] - - for dir in results_path.iterdir(): - if not dir.is_dir(): - continue - m = re.match(DIR_NAME_PATTERN, dir.name) - if not m: - continue - particles.append(int(m[1])) - timem_results_output = timem_results.read_timem_output(dir) - rss, cpu = ( - timem_results.get_timem_metrics_values(timem_results_output) - if timem_results_output - else (None, None) - ) - peak_rss.append(rss) - cpu_usage.append(cpu) - run_evo_ape(dir) - evo_results_path = dir / EVO_RESULTS_FILE_NAME - stats = read_evo_stats(evo_results_path) - ape_rmse.append(stats["rmse"]) - ape_max.append(stats["max"]) - ape_mean.append(stats["mean"]) - ape_std.append(stats["std"]) - ape_median.append(stats["median"]) - terminal_output_path = dir / TERMINAL_OUTPUT_LOG_FILE_NAME - latency_stats = parse_latency_data(terminal_output_path) - latency_min.append(latency_stats['latency_min']) - latency_max.append(latency_stats['latency_max']) - latency_mean.append(latency_stats['latency_mean']) - latency_median.append(latency_stats['latency_median']) - - return ( - pd.DataFrame( - { - 'particles_n': particles, - 'peak_rss': peak_rss, - 'cpu_usage': cpu_usage, - 'ape_rmse': ape_rmse, - 'ape_mean': ape_mean, - 'ape_std': ape_std, - 'ape_median': ape_median, - 'ape_max': ape_max, - 'latency_min': latency_min, - 'latency_max': latency_max, - 'latency_mean': latency_mean, - 'latency_median': latency_median, - } - ) - .set_index('particles_n') - .sort_index() - ) - - -def main(): - arg_parser = argparse.ArgumentParser( - description='Script to compare parameterized run results of beluga with another' - ' implementation (e.g. nav2_amcl, another beluga version, etc).' - ) - arg_parser.add_argument( - '--series', - '-s', - type=Path, - action='append', - help='Folder with parameterized benchmark results', - required=True, - ) - arg_parser.add_argument( - '--label', - '-l', - type=str, - action='append', - help='Label for a series', - required=True, - ) - - arg_parser.add_argument( - '--use-ylog', - action='store_true', - help='Use log scale on y axis', - ) - - default_plots = ['cpu_usage', 'peak_rss', 'ape_rmse', 'latency_median'] - arg_parser.add_argument( - '--plot-names', - type=str, - nargs='*', # 0 or more values expected => creates a list - default=default_plots, - help='List of plots to generate. Default: ' + ', '.join(default_plots), - ) - - arg_parser.add_argument( - '--save-csv', - type=str, - action='store', - help='Instead of plotting, save all the results to a csv file', - ) - - args = arg_parser.parse_args() - - assert len(args.series) == len(args.label), 'Number of series and labels must match' - - color_gen = cycle( - [ - 'red', - 'green', - 'blue', - 'brown', - 'purple', - 'gray', - 'orange', - ] - ) - - if args.save_csv: - # create list of dataframes with no filtered columns - series = [ - create_parameterized_series(series).add_prefix(label + '_') - for label, series in zip(args.label, args.series) - ] - # leave only the index - full_table = series[0].filter(items=[]) - # concatenate data in a single table - for s in series: - full_table = pd.concat([full_table, s], axis=1) - print('Saving to CSV file: ', args.save_csv) - full_table.to_csv(args.save_csv) - return 0 - - series = [ - create_parameterized_series(series) - .filter(items=args.plot_names) - .add_prefix(label + '_') - for label, series in zip(args.label, args.series) - ] - - print('Plotting data...') - - marker_gen = cycle('o^sDvP*') - - ax = series[0].plot( - subplots=True, - color=next(color_gen), - marker=next(marker_gen), - linestyle='dashed', - ) - - for other_series in series[1:]: - other_series.plot( - ax=ax, - subplots=True, - color=next(color_gen), - marker=next(marker_gen), - linestyle='dashed', - ) - - for ax in plt.gcf().axes: - ax.set_xscale('log') - if args.use_ylog: - ax.set_yscale('log') - ax.grid(True, which="both", ls="-") - ax.legend(fontsize='small', loc='upper left', bbox_to_anchor=(1.01, 1)) - current_bounds = ax.get_position().bounds - new_bounds = (0.05, *current_bounds[1:]) - ax.set_position(new_bounds) - current_ylimits = ax.get_ylim() - new_limits = ( - min(0.0, current_ylimits[0]), # include zero below - max(0.0, current_ylimits[1]), # include zero above - ) - ax.set_ylim(new_limits) - - plt.show() diff --git a/beluga_benchmark/beluga_benchmark/exceptions.py b/beluga_benchmark/beluga_benchmark/exceptions.py deleted file mode 100644 index 9c653d65f..000000000 --- a/beluga_benchmark/beluga_benchmark/exceptions.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2023 Ekumen, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class ScriptError(Exception): - pass diff --git a/beluga_benchmark/beluga_benchmark/timem_results.py b/beluga_benchmark/beluga_benchmark/timem_results.py deleted file mode 100644 index a2f752ea6..000000000 --- a/beluga_benchmark/beluga_benchmark/timem_results.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2023 Ekumen, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from .exceptions import ScriptError - -DEFAULT_FILE_NAME = "timem-output.json" - - -def read_timem_output(dir_path: Path): - file = dir_path / DEFAULT_FILE_NAME - try: - with open(file, mode='r') as f: - output = json.load(f) - except (FileExistsError, FileNotFoundError) as ex: - raise ScriptError(f"Failed to open file '{file.absolute()}': {ex}") - except json.decoder.JSONDecodeError: - return None - return output['timemory']['timem'][0] - - -def get_value_with_unit(data): - return f"{data['value']:.2f}{data['unit_repr']}" - - -def get_timem_metrics(output): - return output['peak_rss'], output['cpu_util'] - - -def get_timem_metrics_values(output): - return tuple(data['value'] for data in get_timem_metrics(output)) - - -def get_timem_metrics_with_units(output): - return tuple(get_value_with_unit(data) for data in get_timem_metrics(output)) - - -def create_timeseries(output): - data = output['wall_clock'] - elapsed_time = data['value'] - time = np.arange(0.0, elapsed_time, 0.2) - wall_clock_to_time_offset = output['history'][0]['wall_clock']['value'] - wall_clock_to_time_scale = elapsed_time / ( - output['history'][-1]['wall_clock']['value'] - wall_clock_to_time_offset - ) - time = [] - rss = [] - virtual_memory = [] - for sample in output['history']: - time.append( - (sample['wall_clock']['value'] - wall_clock_to_time_offset) - * wall_clock_to_time_scale - ) - rss.append(sample['page_rss']['value']) - virtual_memory.append(sample['virtual_memory']['value']) - - series = pd.DataFrame( - {'time': time, 'rss': rss, 'virtual_memory': virtual_memory} - ).set_index('time') - return series - - -def main(): - arg_parser = argparse.ArgumentParser( - description='Script to postprocess timem results.' - ) - arg_parser.add_argument( - 'dir', help='Directory with timem-output.json file', type=Path - ) - args = arg_parser.parse_args() - dir_path = Path(args.dir) - name = dir_path.name - output = read_timem_output(dir_path) - peak_rss_str, cpu_usage_str = get_timem_metrics_with_units(output) - series = create_timeseries(output) - elapsed_time_str = get_value_with_unit(output['wall_clock']) - print(f"timem metrics for run '{name}':") - print(f"\telapsed_time: {elapsed_time_str}") - print(f"\tcpu_usage: {cpu_usage_str}") - print(f"\tpeak_rss: {peak_rss_str}") - series.plot(subplots=True) - plt.show() diff --git a/beluga_benchmark/config/qos_override.yml b/beluga_benchmark/config/qos_override.yml new file mode 100644 index 000000000..48cc9b00c --- /dev/null +++ b/beluga_benchmark/config/qos_override.yml @@ -0,0 +1,6 @@ +# durability_override.yml +# Need to override static transform QoS, to reflect that is a latched topic. +# This is not handled properly by ROS1 to ROS2 bag converting tools. +/tf_static: + durability: transient_local + history: keep_all diff --git a/beluga_benchmark/docs/BENCHMARKING.md b/beluga_benchmark/docs/BENCHMARKING.md index 75a8f05bf..25c5b6e10 100644 --- a/beluga_benchmark/docs/BENCHMARKING.md +++ b/beluga_benchmark/docs/BENCHMARKING.md @@ -1,105 +1,29 @@ # How to benchmark Beluga -## Run rosbag benchmark parameterizing the number of particles +1. [Setup a development environment](../../DEVELOPING.md#environment). -This script will run the benchmark with the specified number of particles and record: +2. Build benchmarks and source them: -- The `timem` output: CPU usage, RSS memory, virtual memory. -- A rosbag with the reference and estimated trajectories. -- Perf events (optionally). - -To run, use: - -```bash -ros2 run beluga_benchmark parameterized_run -``` - -The results of the different runs will be stored in folders named `benchmark_${N_PARTICLES}_particles_output`, where `N_PARTICLES` are the numbers specified in the above command. - -To run the same experiment using another AMCL node, e.g. nav2, use: - -```bash -ros2 run beluga_benchmark parameterized_run --package nav2_amcl --executable amcl +```sh +colcon build --packages-up-to beluga_benchmark +source install/setup.bash ``` -For other options, e.g. using a different rosbag, see: +3. Pull benchmark datasets: -```bash -ros2 run beluga_benchmark parameterized_run --help +```sh +mkdir -p src/beluga_benchmark/playground/datasets +# TODO(hidmic): bring datasets from where? ``` -## Visualizing timem results of one run +4. Run a benchmark e.g.: -Use the following command: - -``` -ros2 run beluga_benchmark timem_results +```sh +pushd src/beluga_benchmark/playground +ros2 run beluga_benchmark nominal.robot ``` -Where the specified directory needs to have a `timem-output.json` file, e.g. the output directory generated by `parameterized_run`. -The script will print cpu usage, peak rss and elapsed time. -It will also plot virtual and RSS memory over time. - -## Visualizing estimated trajectory APE of one run - -Use the following command: - -``` -evo_ape bag2 /odometry/ground_truth /pose -p +5. Inspect benchmark results: +```sh +xdg-open nominal/report/latex/report.pdf ``` - -For `nav2_amcl`, replace `/pose` with `/amcl_pose`. -The bagfiles generated by `parameterized_run` can be found in the generated output directories for each run. -This will print APE metrics (mean, median, max, std, rmse, etc), and also plot APE over time. - -## Comparing parameterized runs - -The following command allows to compare the results of different benchmarking runs in a single plot for each metric being measured. - -This can be used to compare different `beluga_amcl` and/or `nav2_amcl` runs, or to compare the same node with different base configuration settings. - -The command is -``` -ros2 run beluga_benchmark compare_results -s -l -s -l ... -``` -where `PATH1` and `PATH2` are the paths to the output directories of the benchmarking runs to compare, and `LABEL1` and `LABEL2` are the labels to use in the plot for each of them. Any number of runs can be added in the same plot by providing additional `-s -l