From f1b1fbb9327f50f3abe21e807650e56b0e8bada5 Mon Sep 17 00:00:00 2001 From: "c.dragomir" Date: Sun, 3 Mar 2024 17:22:34 +0200 Subject: [PATCH 1/4] fixed median_freq issues, integrated rosdiscover, both modes tested --- .gitignore | 3 + README.md | 37 +++++++- extractor.py | 99 +++++++++++++++++++--- graph_merger.py | 43 ++++++++++ rosdiscover_configs/autorally.yml | 9 ++ rosdiscover_configs/fetch.yml | 7 ++ rosdiscover_configs/turtlebot3-slam.yml | 13 +++ rosdiscover_wrapper.py | 107 ++++++++++++++++++++++++ src/extractor/bag_extract.py | 31 +++++-- src/extractor/db3_extract.py | 9 +- src/extractor/functions.py | 20 ++++- src/extractor/main.py | 24 +++--- src/extractor/mcap_extract.py | 7 +- 13 files changed, 365 insertions(+), 44 deletions(-) create mode 100644 .gitignore create mode 100644 graph_merger.py create mode 100644 rosdiscover_configs/autorally.yml create mode 100644 rosdiscover_configs/fetch.yml create mode 100644 rosdiscover_configs/turtlebot3-slam.yml create mode 100644 rosdiscover_wrapper.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f041a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +metrics/* +__pycache__/ +graphs/* \ No newline at end of file diff --git a/README.md b/README.md index 37db735..a4d33b1 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,30 @@ The following figure illustrates the 3-phases approach to extract computation gr ``` ## Installation -Note that it is not necessary for our architecture extractor, which is independent of platforms. However, it requires a few dependencies, solved by the following commands: +### Installing ROSDiscover +Installing rosdiscover has some prerequisites needed for the static code analysis we need to install these first. + +#### Build docker-llvm +Build the [docker-llvm](https://github.com/ChrisTimperley/docker-llvm) image image. This will build the llvm compiler used to build rosdicover-extract-cxx. This might take a while. + +#### Build rosdiscover-extract-cxx +* Clone [the repo](https://github.com/cmu-rss-lab/rosdiscover-cxx-recover) +* Add submodules with their correct versions + * `cd extern` + * `git clone https://github.com/fmtlib/fmt && cd fmt && git checkout d141cdbeb0fb422a3fb7173b285fd38e0d1772dc && cd ..` + * `git clone https://github.com/nlohmann/json nlohmann_json && cd nlohmann_json && git checkout db78ac1d7716f56fc9f1b030b715f872f93964e4 && cd ..` + * `git clone https://github.com/mapbox/variant/ && cd variant && git checkout a5a79a594f39d705a7ef969f54a0743516f0bc6d && cd ../../` +* Build the image + * `cd docker && sudo make install` +After following these steps you will have a docker volume called rosdiscover-cxx-extract-opt ready to be mounted and used by rosdiscover. You can check if it was properly installed by running sudo docker volume ls. + +#### Install the rosdiscover python package +* `pip3 install roswire` +* `git clone https://github.com/cmu-rss-lab/rosdiscover rosdiscover` +* `cd rosdiscover` +* `pip install -e .` + +#### Install the rosbag-extractor dependencies: ``` $ pip3 install -r ./requirements.txt @@ -31,11 +54,20 @@ $ sudo apt install graphviz ``` * If the requirements list is/becomes broken, do not hesitate to pull request the necessary updates. -Then, just run the extraction script on a bag file: + +Now you are finally ready to use the extraction script on a bag file: + ``` $ python3 extractor.py [-h] <-v ROS_VERSION> [-s START_TIME] [-e END_TIME] <-f FILE_PATH> <-ft FILE_TYPE> [-i INPUT] [-ts TIME_SPACE] ``` + parser.add_argument('-rd', '--use-ros-discover', help='Option to run rosdiscover', required=False, type=bool) + parser.add_argument('-rdc', '--ros-discover-config', help='Path to the rosdiscover config', required=False, type=str) + parser.add_argument('-rds', '--node-input-strategy', help='either node-provider or graph-merge', required=False, type=str) + +If you would like to use rosbag extractor with rosdiscover support please you can use the following flags +* `-rd / --use-ros-discover` - Enable the usage of rosdiscover for node input + ##### Example Here, we provide an example with a very simple ROS 2 bag file: @@ -87,3 +119,4 @@ The corresponding metric as follows can be found in the ``metrics`` directory. ``` + \ No newline at end of file diff --git a/extractor.py b/extractor.py index 91cc84b..6afebf5 100644 --- a/extractor.py +++ b/extractor.py @@ -1,7 +1,10 @@ import os, sys import argparse from src.extractor import main - +from rosdiscover_wrapper import * +import pandas as pd +from graph_merger import * +from graphviz import Source def check_files_extension(folder_path, extension): for filename in os.listdir(folder_path): @@ -10,7 +13,7 @@ def check_files_extension(folder_path, extension): return False -def run_extractor(ros_version, start_time, end_time, file_path, filetype, input, time_space): +def run_extractor(ros_version, start_time, end_time, file_path, filetype, input, time_space, rosdiscover): print(">>>> Bag File Extractor <<<<") if start_time is None: @@ -20,19 +23,19 @@ def run_extractor(ros_version, start_time, end_time, file_path, filetype, input, if ros_version == 'ros1': if filetype == 'bag': - main.extractor(start_time, end_time, file_path, filetype, input, time_space) + graph_path = main.extractor(start_time, end_time, file_path, filetype, input, time_space, rosdiscover = rosdiscover) else: print("ROS1 only has filetype `bag`") sys.exit() elif ros_version == 'ros2': if filetype == 'db3': if check_files_extension(file_path, '.db3'): - main.extractor(start_time, end_time, file_path, filetype, input, time_space) + graph_path = main.extractor(start_time, end_time, file_path, filetype, input, time_space, rosdiscover = rosdiscover) else: print("Cannot find the correct file with the input filetype: " + filetype) elif filetype == 'mcap': if check_files_extension(file_path, '.mcap'): - main.extractor(start_time, end_time, file_path, filetype, input, time_space) + graph_path = main.extractor(start_time, end_time, file_path, filetype, input, time_space, rosdiscover = rosdiscover) else: print("Cannot find the correct file with the input filetype: " + filetype) else: @@ -41,7 +44,30 @@ def run_extractor(ros_version, start_time, end_time, file_path, filetype, input, print("ROS version is unknown") print(">>>>>>>>>>>>>><<<<<<<<<<<<<<<") + return graph_path + + +def convert_dot_to_pdf(dot_file_path, pdf_output_path): + """ + Reads a DOT file, displays the graph, and saves it as a PDF. + + Parameters: + - dot_file_path: str, the path to the DOT file. + - pdf_output_path: str, the path where the PDF file will be saved. + """ + print(pdf_output_path) + # Read the DOT file + with open(dot_file_path, 'r') as file: + dot_content = file.read() + + # Create a Source object from the DOT content + dot_graph = Source(dot_content) + # Optionally, view the graph (this will open a viewer window if supported) + dot_graph.view() + + # Save the graph to a PDF file (cleanup=True removes intermediate files) + dot_graph.render(pdf_output_path, format='pdf', cleanup=True) def run(): # Arguments definition and management @@ -53,15 +79,64 @@ def run(): parser.add_argument('-ft', '--file_type', help='Bag file type: bag, db3 or mcap', required=True, type=str) parser.add_argument('-i', '--input', help='Path to file containing nodes information', required=False, type=str) parser.add_argument('-ts', '--time_space', help='Time in second to generate a series of interconnected graphs', required=False, type=str) + parser.add_argument('-rd', '--use-ros-discover', help='Option to run rosdiscover', required=False, type=bool) + parser.add_argument('-rdc', '--ros-discover-config', help='Path to the rosdiscover config', required=False, type=str) + parser.add_argument('-rds', '--node-input-strategy', help='either node-provider or graph-merge', required=False, type=str) options = parser.parse_args() - run_extractor(options.ros_version, - options.start_time, - options.end_time, - options.file_path, - options.file_type, - options.input, - options.time_space) + if options.use_ros_discover is not None: + if options.ros_discover_config is not None: + if options.node_input_strategy is not None: + csv_output_file = f"{options.file_path}/rosdiscover_output.csv" + print("Running ROS Discover",options.ros_discover_config, "with the", options.node_input_strategy, "strategy") + run_rosdiscover(options.ros_discover_config, csv_output_file) + + + if str(options.node_input_strategy) == "graph-merge": + base_graph_path = run_extractor(options.ros_version, + options.start_time, + options.end_time, + options.file_path, + options.file_type, + options.input, + options.time_space, + rosdiscover = True) + + rosdiscover_graph_path = create_graph(path=csv_output_file, format= "dot") + convert_dot_to_pdf(rosdiscover_graph_path, f"{rosdiscover_graph_path}.pdf") + rosdiscover_graph_path = rosdiscover_graph_path + ".dot" + + print("Base graph path: ", base_graph_path) + graph_folder_path = os.path.dirname(os.path.realpath(base_graph_path)) + print("Rosdiscover graph path: ", rosdiscover_graph_path) + + base_graph = parse_dotfile(base_graph_path) + rosdiscover_graph = parse_dotfile(rosdiscover_graph_path) + merged_graph = merge_and_highlight(base_graph, rosdiscover_graph) + export_to_dot(merged_graph, graph_folder_path + "/merged_graph.dot") + convert_dot_to_pdf(graph_folder_path + "/merged_graph.dot", graph_folder_path + "/merged_graph.pdf") + + + elif str(options.node_input_strategy) == "node-provider": + graph_path = run_extractor(options.ros_version, + options.start_time, + options.end_time, + options.file_path, + options.file_type, + csv_output_file, + options.time_space, + rosdiscover = True) + print(graph_path) + + else: + print("Node strategy is not node-provider or graph-merge") + else: + print("Please provide a strategy for the node input, can be either node-provider or graph-merge") + else: + print("Please provide a config file for ROSDiscover") + sys.exit() + + if __name__ == "__main__": diff --git a/graph_merger.py b/graph_merger.py new file mode 100644 index 0000000..15d53d4 --- /dev/null +++ b/graph_merger.py @@ -0,0 +1,43 @@ +import pydot + +def parse_dotfile(file_path): + graphs = pydot.graph_from_dot_file(file_path) + if graphs: + return graphs[0] + else: + return None + +def merge_and_highlight(graph1, graph2): + # Initialize the merged graph + merged_graph = pydot.Dot(graph_type='digraph') + + # Add all nodes and edges from the first graph + for node in graph1.get_nodes(): + merged_graph.add_node(node) + for edge in graph1.get_edges(): + merged_graph.add_edge(edge) + + # Keep track of existing nodes and edges for comparison + existing_nodes = set(node.get_name() for node in merged_graph.get_nodes()) + existing_edges = set((edge.get_source(), edge.get_destination()) for edge in merged_graph.get_edges()) + + # Add nodes from the second graph + for node in graph2.get_nodes(): + if node.get_name() not in existing_nodes: + node.set_color('red') # Highlight new nodes in red + merged_graph.add_node(node) + + # Add edges from the second graph, highlight new edges or edges between existing nodes + for edge in graph2.get_edges(): + edge_tuple = (edge.get_source(), edge.get_destination()) + if edge_tuple not in existing_edges: + edge.set_color('red') # Highlight new edges in red + merged_graph.add_edge(edge) + elif edge.get_color() != 'red': # For existing edges not highlighted, ensure they're not colored red + edge.set_color('') # Reset color to default if re-adding an existing edge that's not new + + return merged_graph + +def export_to_dot(merged_graph, output_file): + merged_graph.write_dot(output_file) + print(f"Exported merged graph to {output_file}") \ No newline at end of file diff --git a/rosdiscover_configs/autorally.yml b/rosdiscover_configs/autorally.yml new file mode 100644 index 0000000..12ecb8e --- /dev/null +++ b/rosdiscover_configs/autorally.yml @@ -0,0 +1,9 @@ +image: therobotcooperative/autorally +sources: + - /opt/ros/melodic/setup.bash + - /ros_ws/devel/setup.bash + - /ros_ws/src/autorally/autorally_util/setupEnvLocal.sh +launches: + - /ros_ws/src/autorally/autorally_gazebo/launch/autoRallyTrackGazeboSim.launch + - /ros_ws/src/autorally/autorally_control/launch/waypointFollower.launch + - /ros_ws/src/autorally/autorally_control/launch/constantSpeedController.launch diff --git a/rosdiscover_configs/fetch.yml b/rosdiscover_configs/fetch.yml new file mode 100644 index 0000000..dab5a17 --- /dev/null +++ b/rosdiscover_configs/fetch.yml @@ -0,0 +1,7 @@ +image: therobotcooperative/fetch +sources: +- /opt/ros/melodic/setup.bash +- /ros_ws/devel/setup.bash +launches: +- /ros_ws/src/fetch_gazebo/fetch_gazebo/launch/playground.launch +- /ros_ws/src/fetch_gazebo/fetch_gazebo_demo/launch/demo.launch diff --git a/rosdiscover_configs/turtlebot3-slam.yml b/rosdiscover_configs/turtlebot3-slam.yml new file mode 100644 index 0000000..1aab081 --- /dev/null +++ b/rosdiscover_configs/turtlebot3-slam.yml @@ -0,0 +1,13 @@ +image: cristidragomir97/turtlebot3-melodic +sources: +- /opt/ros/melodic/setup.bash +- /turtlebot3_ws/devel/setup.bash +environment: + TURTLEBOT3_MODEL: burger +launches: +- filename: /turtlebot3_ws/src/turtlebot3_simulations/turtlebot3_gazebo/launch/turtlebot3_house.launch +- filename: /turtlebot3_ws/src/turtlebot3/turtlebot3_teleop/launch/turtlebot3_teleop_key.launch +- filename: /turtlebot3_ws/src/turtlebot3/turtlebot3_slam/launch/turtlebot3_slam.launch + slam_methods: gmapping + + diff --git a/rosdiscover_wrapper.py b/rosdiscover_wrapper.py new file mode 100644 index 0000000..cdd2a3a --- /dev/null +++ b/rosdiscover_wrapper.py @@ -0,0 +1,107 @@ +import subprocess, re, os, csv, yaml, sys +import pandas as pd +from graphviz import Digraph +from ruamel.yaml import YAML +import re + +def run_script_and_capture_output(script_path, * args): + timestamp_pattern = r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}" + warning_pattern = r"^Warning: " + + # List to hold the output lines + output_lines = "" + + # Construct the command with the script path and its arguments + command = [script_path] + list(args) + + # Run the script and capture the output + with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as proc: + for line in proc.stdout: + # Check if the line starts with a timestamp + if not re.match(timestamp_pattern, line): + if not re.match(warning_pattern, line): + output_lines += line + print(line, end='') + # Join the lines back into a single string + + + # Use ruamel.yaml to load the YAML preserving the formatting + yaml = YAML() + yaml.preserve_quotes = True + data = yaml.load(output_lines) + + return data + +def format_topics_as_array(topics): + """Formats the list of topics as a string array, with each topic in single quotes.""" + quoted_topics = [f"'{topic}'" for topic in topics] # Surround each topic with single quotes + return f"\"[{','.join(quoted_topics)}]\"" if topics else "[]" + +def create_graph(path, format="dot"): + print(path) + df = pd.read_csv(path) + out_file = os.path.dirname(path) + "/rosdiscover_graph" + print(path, out_file) + # Initialize a Digraph object + dot = Digraph(comment='ROS Nodes and Topics') + + # Add nodes and edges + for index, row in df.iterrows(): + node_name = row['Name'] + publishes = eval(row['Publish']) if row['Publish'] not in [None, '[]', ''] else [] + subscribes = eval(row['Subscribe']) if row['Subscribe'] not in [None, '[]', ''] else [] + + # Add the node for the ROS node + dot.node(node_name, node_name, shape='ellipse') + + # Add nodes and edges for publish topics + for topic in publishes: + dot.node(topic, topic, shape='box') + dot.edge(node_name, topic, label='') + + # Add nodes and edges for subscribe topics + for topic in subscribes: + dot.node(topic, topic, shape='box') + dot.edge(topic, node_name, label='') + + # Render the graph to a file (e.g., output.pdf) + dot.render(out_file, view=False, format=format) + return out_file + +def remove_whitespace_from_csv(file_path): + # Open the file, read its contents as a string + with open(file_path, 'r') as file: + content = file.read() + + # Remove all spaces from the content + modified_content = content.replace(" ", "") + + # Write the modified content back to the file + with open(file_path, 'w') as file: + file.write(modified_content) + +def run_rosdiscover(config_path, out_file): + nodes = run_script_and_capture_output("rosdiscover", 'launch', config_path) + + # Prepare data for CSV + csv_data = [] + + + for node in nodes: + name = "/" + node.get('name', '') + publishes = format_topics_as_array([pub['name'] for pub in node.get('pubs', [])]) + subscribes = format_topics_as_array([sub['name'] for sub in node.get('subs', [])]) + node_start = '' + node_end = '' + print(name, publishes, subscribes, node_start, node_end) + csv_data.append([name, publishes, subscribes, node_start, node_end]) + + + with open(out_file, 'w', newline='') as file: + writer = csv.writer(file, quoting=csv.QUOTE_NONE, escapechar=' ') + writer.writerow(['Name', 'Publish', 'Subscribe', 'Node-start', 'Node-end']) + writer.writerows(csv_data) + + remove_whitespace_from_csv(out_file) + return out_file + diff --git a/src/extractor/bag_extract.py b/src/extractor/bag_extract.py index deb2037..c3a8a18 100644 --- a/src/extractor/bag_extract.py +++ b/src/extractor/bag_extract.py @@ -22,13 +22,16 @@ def generate_topics(bag, graph, all_topics, metric): stamps = tmp['Stamps'].tolist() period = [s1 - s0 for s1, s0 in zip(stamps[1:], stamps[:-1])] med_period = functions._median(period) - med_freq = round((1.0 / med_period), 2) + if med_period == 0.0: + med_freq = 0.0 + else: + med_freq = round((1.0 / med_period), 2) if str(med_freq) != 'nan': graph.node(topic, topic, {'shape': 'rectangle'}, xlabel=(str(med_freq) + 'Hz')) else: graph.node(topic, topic, {'shape': 'rectangle'}) data = {topic: {'name': topic, - 'start': stamps[1], + 'start': stamps[0], 'end': stamps[-1], 'frequency': med_freq }} @@ -45,13 +48,16 @@ def generate_topics(bag, graph, all_topics, metric): stamps = tmp['Stamps'].tolist() period = [s1 - s0 for s1, s0 in zip(stamps[1:], stamps[:-1])] med_period = functions._median(period) - med_freq = round((1.0 / med_period), 2) + if med_period == 0.0: + med_freq = 0.0 + else: + med_freq = round((1.0 / med_period), 2) if str(med_freq) != 'nan': sub_topic.node(topic, topic, {'shape': 'rectangle'}, xlabel=(str(med_freq) + 'Hz')) else: sub_topic.node(topic, topic, {'shape': 'rectangle'}) data = {topic: {'name': topic, - 'start': stamps[1], + 'start': stamps[0], 'end': stamps[-1], 'frequency': med_freq }} @@ -94,7 +100,7 @@ def generate_edges(graph, rosout_info, topics, nodes, metric): metric['Nodes'][subscriber]['#publisher'] += 1 -def extract_graph(bag, topics, external_nodes, rosout_info, metric, graph_n): +def extract_graph(bag, topics, external_nodes, rosout_info, metric, graph_n, rosdiscover = False): graph = Digraph(name=bag+graph_n) # initialize the metric @@ -141,15 +147,20 @@ def extract_graph(bag, topics, external_nodes, rosout_info, metric, graph_n): functions.generate_edges_external(bag, external_nodes, graph, metric) # save graph - functions.save_graph(bag, graph, graph_n, "ros1") + graph_path = functions.save_graph(bag, graph, graph_n, "ros1", ) # save metric functions.save_metric(metric, bag, graph_n) # view graph - graph.unflatten(stagger=3, fanout=True).view() + if not rosdiscover: + graph.unflatten(stagger=3, fanout=True).view() + + return graph_path -def main(bagfolder, start_t, end_t, input_file, graph_n): + + +def main(bagfolder, start_t, end_t, input_file, graph_n, rosdiscover = False): bagfile = get_file_name(bagfolder) bag = bagfile.replace('.bag', '') @@ -205,7 +216,7 @@ def main(bagfolder, start_t, end_t, input_file, graph_n): metric['Start'] = start_t metric['End'] = end_t - extract_graph(bag, topics, nodes, rosout_info, metric, graph_n) + graph_path = extract_graph(bag, topics, nodes, rosout_info, metric, graph_n) # save metric directory = 'metrics/' @@ -214,5 +225,7 @@ def main(bagfolder, start_t, end_t, input_file, graph_n): with open(metric_path, 'w') as json_file: json.dump(metric, json_file, indent=4) + return graph_path + # if __name__ == "__main__": # main() diff --git a/src/extractor/db3_extract.py b/src/extractor/db3_extract.py index f33a426..c28dc73 100644 --- a/src/extractor/db3_extract.py +++ b/src/extractor/db3_extract.py @@ -5,7 +5,7 @@ from src.extractor import functions -def main(bagfolder, start_t, end_t, input_file, graph_n): +def main(bagfolder, start_t, end_t, input_file, graph_n, rosdiscover = False): graph = Digraph(name=bagfolder+graph_n) graph.graph_attr["rankdir"] = "LR" @@ -47,10 +47,13 @@ def main(bagfolder, start_t, end_t, input_file, graph_n): functions.create_graph(bagfolder, graph, topics, nodes, graph_n, metric) # save graph - functions.save_graph(bagfolder, graph, graph_n, "ros2") + graph_path = functions.save_graph(bagfolder, graph, graph_n, "ros2") # view graph - graph.unflatten(stagger=5, fanout=True).view() + if not rosdiscover: + graph.unflatten(stagger=5, fanout=True).view() + + return graph_path # if __name__ == '__main__': # main(bagfolder, file, start, end, input) diff --git a/src/extractor/functions.py b/src/extractor/functions.py index 8d08643..4a3dc9b 100644 --- a/src/extractor/functions.py +++ b/src/extractor/functions.py @@ -59,8 +59,12 @@ def _median(values): def get_freq(stamps): period = [s1 - s0 for s1, s0 in zip(stamps[1:], stamps[:-1])] med_period = _median(period) - med_freq = round((1.0 / med_period), 2) - return med_freq + + if med_period == 0.0: + return 0.0 + else: + med_freq = round((1.0 / med_period), 2) + return med_freq def get_mean_freq(stamps): @@ -97,14 +101,19 @@ def generate_topics(bagfolder, graph, topics, graph_n, metric): stamps = tmp['Stamps'].tolist() period = [s1 - s0 for s1, s0 in zip(stamps[1:], stamps[:-1])] med_period = _median(period) - med_freq = round((1.0 / med_period), 2) + if med_period == 0.0: + med_freq = 0.0 + else: + med_freq = round((1.0 / med_period), 2) + + if str(med_freq) != 'nan': graph.node(topic, topic, {'shape': 'rectangle'}, xlabel=(str(med_freq)+'Hz')) else: graph.node(topic, topic, {'shape': 'rectangle'}) data = {topic: {'name': topic, - 'start': stamps[1], + 'start': stamps[0], 'end': stamps[-1], 'frequency': med_freq }} @@ -231,3 +240,6 @@ def save_graph(bagfolder, graph, graph_n, ros_v): dot_file = "graphs/ros2/" + bagname + '/' + bagname + '_' + graph_n + '.dot' with open(dot_file, 'w') as dot_file: dot_file.write(graph.source) + + path = "graphs/" + ros_v + "/" + bagname + '/' + bagname + '_' + graph_n + '.dot' + return path diff --git a/src/extractor/main.py b/src/extractor/main.py index c455c02..3d1416c 100644 --- a/src/extractor/main.py +++ b/src/extractor/main.py @@ -47,7 +47,7 @@ def check_time_range(start, bag_start, end, bag_end): return start_t, end_t -def extractor(start, end, path_to_file, filetype, input_file, time_space): +def extractor(start, end, path_to_file, filetype, input_file, time_space, rosdiscover = False): if filetype == 'bag': bagfile = path_to_file + '/' + path_to_file.split('/')[-1] + ".bag" @@ -58,24 +58,24 @@ def extractor(start, end, path_to_file, filetype, input_file, time_space): if time_space is not None: start_t_spaced = start_t - graph_n = 0 + graph_n = 2 time_space = float(time_space) while start_t_spaced + time_space < bag_end: print("The extraction of graph " + str(graph_n) + " STARTS at", start_t_spaced, "and ENDS at", start_t_spaced + time_space, "\nThe duration of bag is: " + str(time_space), 'seconds') - bag.main(path_to_file, start_t_spaced, start_t_spaced + time_space, input_file, str(graph_n)) + graph_path = bag.main(path_to_file, start_t_spaced, start_t_spaced + time_space, input_file, str(graph_n), rosdiscover=rosdiscover) start_t_spaced += time_space graph_n += 1 # last graph print("The extraction of graph " + str(graph_n) + " STARTS at", start_t_spaced, "and ENDS at", end_t, "\nThe duration of bag is: " + str(end_t - start_t_spaced), 'seconds') - bag.main(path_to_file, start_t_spaced, end_t, input_file, str(graph_n)) + graph_path = bag.main(path_to_file, start_t_spaced, end_t, input_file, str(graph_n), rosdiscover=rosdiscover) else: graph_n = 0 print("The extraction STARTS at", start_t, "and ENDS at", end_t, "\nThe duration of bag is: " + str(end_t - start_t), 'seconds') - bag.main(path_to_file, start_t, end_t, input_file, str(graph_n)) + graph_path = bag.main(path_to_file, start_t, end_t, input_file, str(graph_n), rosdiscover=rosdiscover) elif filetype == 'db3': with Reader(path_to_file) as reader: bag_start = reader.start_time / 1000000000 @@ -89,19 +89,19 @@ def extractor(start, end, path_to_file, filetype, input_file, time_space): while start_t_spaced+time_space < bag_end: print("The extraction of graph " + str(graph_n) + " STARTS at", start_t_spaced, "and ENDS at", start_t_spaced+time_space, "\nThe duration of bag is: " + str(time_space), 'seconds') - db3.main(path_to_file, start_t_spaced, start_t_spaced+time_space, input_file, str(graph_n)) + graph_path = db3.main(path_to_file, start_t_spaced, start_t_spaced+time_space, input_file, str(graph_n), rosdiscover=rosdiscover) start_t_spaced += time_space graph_n += 1 # last graph print("The extraction of graph " + str(graph_n) + " STARTS at", start_t_spaced, "and ENDS at", end_t, "\nThe duration of bag is: " + str(end_t-start_t_spaced), 'seconds') - db3.main(path_to_file, start_t_spaced, end_t, input_file, str(graph_n)) + graph_path = db3.main(path_to_file, start_t_spaced, end_t, input_file, str(graph_n), rosdiscover=rosdiscover) else: graph_n = 0 print("The extraction STARTS at", start_t, "and ENDS at", end_t, "\nThe duration of bag is: " + str(end_t-start_t), 'seconds') - db3.main(path_to_file, start_t, end_t, input_file, str(graph_n)) + graph_path = db3.main(path_to_file, start_t, end_t, input_file, str(graph_n), rosdiscover = rosdiscover) else: file_mcap = get_mcap_file_name(path_to_file) @@ -132,20 +132,20 @@ def extractor(start, end, path_to_file, filetype, input_file, time_space): while start_t_spaced+time_space < bag_end: print("The extraction of graph " + str(graph_n) + " STARTS at", start_t_spaced, "and ENDS at", start_t_spaced+time_space, "\nThe duration of bag is: " + str(time_space), 'seconds') - mcap.main(path_to_file, file_mcap, start_t_spaced, start_t_spaced+time_space, input_file, str(graph_n)) + graph_path = mcap.main(path_to_file, file_mcap, start_t_spaced, start_t_spaced+time_space, input_file, str(graph_n), rosdiscover=rosdiscover) start_t_spaced += time_space graph_n += 1 # last graph print("The extraction of graph " + str(graph_n) + " STARTS at", start_t_spaced, "and ENDS at", end_t, "\nThe duration of bag is: " + str(end_t-start_t_spaced), 'seconds') - mcap.main(path_to_file, file_mcap, start_t_spaced, end_t, input_file, str(graph_n)) + graph_path = mcap.main(path_to_file, file_mcap, start_t_spaced, end_t, input_file, str(graph_n), rosdiscover=rosdiscover) else: graph_n = 0 print("The extraction STARTS at", start_t, "and ENDS at", end_t, "\nThe duration of bag is: " + str(end_t-start_t), 'seconds') - mcap.main(path_to_file, file_mcap, start_t, end_t, input_file, str(graph_n)) - + graph_path = mcap.main(path_to_file, file_mcap, start_t, end_t, input_file, str(graph_n), rosdiscover=rosdiscover) + return graph_path # if __name__ == '__main__': # extractor() diff --git a/src/extractor/mcap_extract.py b/src/extractor/mcap_extract.py index ade0657..207d72f 100644 --- a/src/extractor/mcap_extract.py +++ b/src/extractor/mcap_extract.py @@ -54,10 +54,13 @@ def main(bagfolder, file, start_t, end_t, input_file, graph_n): functions.create_graph(bagfolder, graph, topics, nodes, graph_n, metric) # save graph - functions.save_graph(bagfolder, graph, graph_n, "ros2") + graph_path = functions.save_graph(bagfolder, graph, graph_n, "ros2") # view graph - graph.unflatten(stagger=5, fanout=True).view() + if not rosdiscover: + graph.unflatten(stagger=5, fanout=True).view() + + return graph_path # if __name__ == '__main__': # main(bagfolder, file, start, end, input) From a9b1b6e4c98ee7cc1684a276be1416ba69ddec43 Mon Sep 17 00:00:00 2001 From: "c.dragomir" Date: Sun, 3 Mar 2024 17:33:12 +0200 Subject: [PATCH 2/4] forgot the mode where rosdiscover is not used at all --- extractor.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/extractor.py b/extractor.py index 6afebf5..bd6d852 100644 --- a/extractor.py +++ b/extractor.py @@ -132,9 +132,15 @@ def run(): print("Node strategy is not node-provider or graph-merge") else: print("Please provide a strategy for the node input, can be either node-provider or graph-merge") - else: - print("Please provide a config file for ROSDiscover") - sys.exit() + else: + run_extractor(options.ros_version, + options.start_time, + options.end_time, + options.file_path, + options.file_type, + options.input, + options.time_space, + rosdiscover = False) From 94dbf889ccc352358dde8cc242f6b3688a0e6be0 Mon Sep 17 00:00:00 2001 From: cristidragomir97 Date: Sun, 3 Mar 2024 17:33:59 +0200 Subject: [PATCH 3/4] Update README.md --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a4d33b1..d629d06 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Build the [docker-llvm](https://github.com/ChrisTimperley/docker-llvm) image ima * `git clone https://github.com/mapbox/variant/ && cd variant && git checkout a5a79a594f39d705a7ef969f54a0743516f0bc6d && cd ../../` * Build the image * `cd docker && sudo make install` + After following these steps you will have a docker volume called rosdiscover-cxx-extract-opt ready to be mounted and used by rosdiscover. You can check if it was properly installed by running sudo docker volume ls. #### Install the rosdiscover python package @@ -65,10 +66,12 @@ $ python3 extractor.py [-h] <-v ROS_VERSION> [-s START_TIME] [-e END_TIME] <-f F parser.add_argument('-rdc', '--ros-discover-config', help='Path to the rosdiscover config', required=False, type=str) parser.add_argument('-rds', '--node-input-strategy', help='either node-provider or graph-merge', required=False, type=str) -If you would like to use rosbag extractor with rosdiscover support please you can use the following flags +If you would like to use rosbag extractor with rosdiscover support please use the following flags: * `-rd / --use-ros-discover` - Enable the usage of rosdiscover for node input +* `-rdc / --ros-discover-config` - Path to the configuration for rosdiscover. Check out [this example](https://github.com/cristidragomir97/rosbag-extractor/blob/main/rosdiscover_configs/turtlebot3-slam.yml) +* `-rds / --ros-discover-strategy` - The strategy used for integrationg the rosdiscover output. `graph-merge` creates an architecture graph using rosart and another one using rosdiscover and merges them, while `node-provider` uses the input from rosdiscover as an input for rosbag-extractor. -##### Example +## Example Here, we provide an example with a very simple ROS 2 bag file: ``` @@ -119,4 +122,4 @@ The corresponding metric as follows can be found in the ``metrics`` directory. ``` - \ No newline at end of file + From a808b91a7a928296373ef524658ec293e7f6e387 Mon Sep 17 00:00:00 2001 From: cristidragomir97 Date: Sun, 3 Mar 2024 17:36:57 +0200 Subject: [PATCH 4/4] Update requirement.txt --- requirement.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirement.txt b/requirement.txt index f63f250..111c809 100644 --- a/requirement.txt +++ b/requirement.txt @@ -2,3 +2,6 @@ bagpy>=0.4.8 graphviz>=0.20 catkin_pkg>0.5.2 rosbags +pandas=2.2.1 +pydot=2.0.0 +ruamel.yaml=0.18.6